signer/storage,cmd/clef rework clef to support db_storage

cmd/clef/main.go

@@ -130,9 +130,21 @@ var (
 		Name:  "stdio-ui-test",
 		Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.",
 	}
-	vaultFlag = cli.StringFlag{
+	keystoreDBFlag = cli.StringFlag{
-		Name:  "vault",
+		Name:  "keystore-db",
-		Usage: "URI to the user managed database which stores Clef vault data (ruleset & kps)",
+		Usage: "General SQL database name which stores keys, currently supported db type are: mysql, postgres",
+	}
+	keystoreDBDSNFlag = cli.StringFlag{
+		Name:  "keystore-db-dsn",
+		Usage: "Data Source Name (DSN) for keystore-db",
+	}
+	configDBFlag = cli.StringFlag{
+		Name:  "config-db",
+		Usage: "General SQL database name which is used to store configuration key value pairs, currently supported db type are: mysql, postgres",
+	}
+	configDBDSNFlag = cli.StringFlag{
+		Name:  "config-db-dsn",
+		Usage: "Data Source Name (DSN) for config-db",
 	}
 	app         = cli.NewApp()
 	initCommand = cli.Command{
@@ -223,7 +235,10 @@ func init() {
 		ruleFlag,
 		stdiouiFlag,
 		testFlag,
-		vaultFlag,
+		keystoreDBFlag,
+		keystoreDBDSNFlag,
+		configDBFlag,
+		configDBDSNFlag,
 		advancedMode,
 	}
 	app.Action = signer
@@ -431,11 +446,31 @@ func ipcEndpoint(ipcPath, datadir string) string {
 	return ipcPath
 }
 
+func checkFlag(c *cli.Context) error {
+	keystoreDB := c.GlobalString(keystoreDBFlag.Name)
+	keystoreDBDSN := c.GlobalString(keystoreDBDSNFlag.Name)
+	if keystoreDB != "" && keystoreDBDSN == "" {
+		return fmt.Errorf("%s has to be set along with %s", keystoreDBDSNFlag.Name, keystoreDBFlag.Name)
+	}
+
+	configDB := c.GlobalString(configDBFlag.Name)
+	configDBDSN := c.GlobalString(configDBDSNFlag.Name)
+	if configDB != "" && configDBDSN == "" {
+		return fmt.Errorf("%s has to be set along with %s", configDBDSNFlag.Name, configDBFlag.Name)
+	}
+
+	return nil
+}
+
 func signer(c *cli.Context) error {
 	// If we have some unrecognized command, bail out
 	if args := c.Args(); len(args) > 0 {
 		return fmt.Errorf("invalid command: %q", args[0])
 	}
+	// check flag legitimacy
+	if err := checkFlag(c); err != nil {
+		return err
+	}
 	if err := initialize(c); err != nil {
 		return err
 	}
@@ -461,23 +496,36 @@ func signer(c *cli.Context) error {
 	var (
 		api           core.ExternalAPI
 		pwStorage     storage.Storage = &storage.NoStorage{}
+		jsStorage     storage.Storage = &storage.NoStorage{}
+		configStorage storage.Storage = &storage.NoStorage{}
 	)
 
 	configDir := c.GlobalString(configdirFlag.Name)
 	if stretchedKey, err := readMasterKey(c, ui); err != nil {
 		log.Warn("Failed to open master, rules disabled", "err", err)
 	} else {
-		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
-
 		// Generate domain specific keys
 		pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 		jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey)
 		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
 
+		if configDB := c.GlobalString(configDBFlag.Name); configDB != "" {
+			configDBDSN := c.GlobalString(configDBDSNFlag.Name)
+			pwStorage, err = storage.NewDBStorage(pwkey, configDB, configDBDSN, storage.PasswordTable)
+			if err != nil {
+				utils.Fatalf("Could not connect to config db %v, %v", configDB, configDBDSN)
+			}
+			// since we're literally connecting to the same database, so we don't need to check for err for 3 times
+			jsStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.JsTable)
+			configStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.ConfigTable)
+		} else {
+			vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+
 			// Initialize the encrypted storages
 			pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
+			jsStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
-		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
+			configStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
+		}
 
 		// Do we have a rule-file?
 		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {

go.mod

@@ -59,9 +59,10 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+	golang.org/x/mobile v0.0.0-20200123024942-82c397c4c527 // indirect
 	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
-	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7
 	golang.org/x/text v0.3.2
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4

go.sum

@@ -20,6 +20,7 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
@@ -196,15 +197,34 @@ github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20200123024942-82c397c4c527 h1:470B1IvA7JOANiTULBeEJCIcp53VWOsgckscR836cN8=
+golang.org/x/mobile v0.0.0-20200123024942-82c397c4c527/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayERXBdfZjUYoXEf5BTfDfh8=
+golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -213,6 +233,13 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

signer/storage/db_storage.go

@@ -46,6 +46,13 @@ type DBRow struct {
 	val string
 }
 
+// Default table name for storages
+const (
+	PasswordTable = "kps"
+	ConfigTable   = "config"
+	JsTable       = "js"
+)
+
 // NewDBStorage create new database backed storage
 func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DBStorage, error) {
 	// sql.Open only validates the input, but didn't create a connection
@@ -67,6 +74,9 @@ func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DB
 	// set connection limits
 	db.SetMaxOpenConns(5)
 
+	// init table
+	initTable(driverName, tableName, db)
+
 	return &DBStorage{
 		driverName:     driverName,
 		dataSourceName: dataSourceName,
@@ -76,6 +86,38 @@ func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DB
 	}, nil
 }
 
+func initTable(driverName, tableName string, db *sql.DB) error {
+	var err error
+	switch driverName {
+	case "postgres":
+		_, err = db.Exec(fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+	id SERIAL PRIMARY KEY,
+	k VARCHAR(255) UNIQUE NOT NULL,
+	v TEXT NOT NULL
+)
+		`, tableName))
+	case "mysql":
+		_, err = db.Exec(fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+	id INT AUTO_INCREMENT PRIMARY KEY,
+	k VARCHAR(255) UNIQUE NOT NULL, 
+	v TEXT NOT NULL
+)
+		`, tableName))
+	case "sqlite3":
+		_, err = db.Exec(fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+	id INTEGER PRIMARY KEY, 
+	k TEXT, 
+	v TEXT
+)
+		`, tableName))
+	}
+
+	return err
+}
+
 // Put stores a value by key. 0-length keys results in noop.
 func (s *DBStorage) Put(key, value string) {
 	if len(key) == 0 {
@@ -166,10 +208,10 @@ func (s *DBStorage) queryRow(query string, args ...interface{}) (*DBRow, bool, e
 }
 
 var (
-	getSQL    string = "SELECT * FROM tableName WHERE key = ?"
+	getSQL    string = "SELECT * FROM tableName WHERE k = ?"
-	updateSQL string = "UPDATE tableName SET val = ? WHERE key = ?"
+	updateSQL string = "UPDATE tableName SET v = ? WHERE k = ?"
-	insertSQL string = "INSERT INTO tableName (key, val) VALUES (?, ?)"
+	insertSQL string = "INSERT INTO tableName (k, v) VALUES (?, ?)"
-	deleteSQL string = "DELETE FROM tableName WHERE key = ?"
+	deleteSQL string = "DELETE FROM tableName WHERE k = ?"
 )
 
 func (s *DBStorage) formatSQL(sql string) string {
@@ -179,13 +221,9 @@ func (s *DBStorage) formatSQL(sql string) string {
 		for i := 1; i <= params; i++ {
 			sql = strings.Replace(sql, "?", fmt.Sprintf("$%d", i), 1)
 		}
-	case "goracle":
-		params := strings.Count(sql, "?")
-		for i := 1; i <= params; i++ {
-			sql = strings.Replace(sql, "?", fmt.Sprintf(":v%d", i), 1)
-		}
 	default:
 		// for MS SQL Server / MySQL / SQLite
+		// since they're already using ? as placeholder, do nothing
 	}
 
 	return strings.ReplaceAll(sql, "tableName", s.tableName)

signer/storage/db_storage_test.go

@@ -1,7 +1,6 @@
 package storage
 
 import (
-	"fmt"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -17,9 +16,7 @@ var (
 func init() {
 	key = "AES256Key-32Characters1234567890"
 	tmpDir, _ := ioutil.TempDir("", "eth-encrypted-db-storge-test")
-	fmt.Println(tmpDir)
 	ds, _ = NewDBStorage([]byte(key), "sqlite3", filepath.Join(tmpDir, "test.db"), "kps")
-	ds.exec("CREATE TABLE IF NOT EXISTS kps (id INTEGER PRIMARY KEY, key TEXT, val TEXT)")
 }
 
 func TestDBStorage(t *testing.T) {
@@ -55,10 +52,10 @@ func TestSwappedKeysForDBStorage(t *testing.T) {
 
 	// now make a modified copy
 	swap := func() {
-		creds1, _, _ := ds.queryRow("SELECT * FROM kps WHERE key = 'k1'")
+		creds1, _, _ := ds.queryRow("SELECT * FROM kps WHERE k = 'k1'")
-		creds2, _, _ := ds.queryRow("SELECT * FROM kps WHERE key = 'k2'")
+		creds2, _, _ := ds.queryRow("SELECT * FROM kps WHERE k = 'k2'")
-		ds.exec("UPDATE kps SET val = ? WHERE key = ?", creds1.val, "k2")
+		ds.exec("UPDATE kps SET v = ? WHERE k = ?", creds1.val, "k2")
-		ds.exec("UPDATE kps SET val = ? WHERE key = ?", creds2.val, "k1")
+		ds.exec("UPDATE kps SET v = ? WHERE k = ?", creds2.val, "k1")
 	}
 	swap()
 	if v, _ := ds.Get("k1"); v != "" {