From f7f9d1701ed7e05aa85cfabfaa7ddd100d716658 Mon Sep 17 00:00:00 2001 From: Huiyi Li Date: Sun, 26 Jan 2020 09:55:52 -0800 Subject: [PATCH] signer/storage,cmd/clef rework clef to support db_storage --- cmd/clef/main.go | 72 +++++++++++++++++++++++++------ go.mod | 5 ++- go.sum | 27 ++++++++++++ signer/storage/db_storage.go | 56 ++++++++++++++++++++---- signer/storage/db_storage_test.go | 11 ++--- 5 files changed, 141 insertions(+), 30 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 4bfac09bdd..e0cc60d342 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -130,9 +130,21 @@ var ( Name: "stdio-ui-test", Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.", } - vaultFlag = cli.StringFlag{ - Name: "vault", - Usage: "URI to the user managed database which stores Clef vault data (ruleset & kps)", + keystoreDBFlag = cli.StringFlag{ + Name: "keystore-db", + Usage: "General SQL database name which stores keys, currently supported db type are: mysql, postgres", + } + keystoreDBDSNFlag = cli.StringFlag{ + Name: "keystore-db-dsn", + Usage: "Data Source Name (DSN) for keystore-db", + } + configDBFlag = cli.StringFlag{ + Name: "config-db", + Usage: "General SQL database name which is used to store configuration key value pairs, currently supported db type are: mysql, postgres", + } + configDBDSNFlag = cli.StringFlag{ + Name: "config-db-dsn", + Usage: "Data Source Name (DSN) for config-db", } app = cli.NewApp() initCommand = cli.Command{ @@ -223,7 +235,10 @@ func init() { ruleFlag, stdiouiFlag, testFlag, - vaultFlag, + keystoreDBFlag, + keystoreDBDSNFlag, + configDBFlag, + configDBDSNFlag, advancedMode, } app.Action = signer @@ -431,11 +446,31 @@ func ipcEndpoint(ipcPath, datadir string) string { return ipcPath } +func checkFlag(c *cli.Context) error { + keystoreDB := c.GlobalString(keystoreDBFlag.Name) + keystoreDBDSN := c.GlobalString(keystoreDBDSNFlag.Name) + if keystoreDB != "" && keystoreDBDSN == "" { + return fmt.Errorf("%s has to be set along with %s", keystoreDBDSNFlag.Name, keystoreDBFlag.Name) + } + + configDB := c.GlobalString(configDBFlag.Name) + configDBDSN := c.GlobalString(configDBDSNFlag.Name) + if configDB != "" && configDBDSN == "" { + return fmt.Errorf("%s has to be set along with %s", configDBDSNFlag.Name, configDBFlag.Name) + } + + return nil +} + func signer(c *cli.Context) error { // If we have some unrecognized command, bail out if args := c.Args(); len(args) > 0 { return fmt.Errorf("invalid command: %q", args[0]) } + // check flag legitimacy + if err := checkFlag(c); err != nil { + return err + } if err := initialize(c); err != nil { return err } @@ -459,25 +494,38 @@ func signer(c *cli.Context) error { log.Info("Loaded 4byte database", "embeds", embeds, "locals", locals, "local", fourByteLocal) var ( - api core.ExternalAPI - pwStorage storage.Storage = &storage.NoStorage{} + api core.ExternalAPI + pwStorage storage.Storage = &storage.NoStorage{} + jsStorage storage.Storage = &storage.NoStorage{} + configStorage storage.Storage = &storage.NoStorage{} ) configDir := c.GlobalString(configdirFlag.Name) if stretchedKey, err := readMasterKey(c, ui); err != nil { log.Warn("Failed to open master, rules disabled", "err", err) } else { - vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) - // Generate domain specific keys pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey) confkey := crypto.Keccak256([]byte("config"), stretchedKey) - // Initialize the encrypted storages - pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) - jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey) - configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) + if configDB := c.GlobalString(configDBFlag.Name); configDB != "" { + configDBDSN := c.GlobalString(configDBDSNFlag.Name) + pwStorage, err = storage.NewDBStorage(pwkey, configDB, configDBDSN, storage.PasswordTable) + if err != nil { + utils.Fatalf("Could not connect to config db %v, %v", configDB, configDBDSN) + } + // since we're literally connecting to the same database, so we don't need to check for err for 3 times + jsStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.JsTable) + configStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.ConfigTable) + } else { + vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) + + // Initialize the encrypted storages + pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) + jsStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey) + configStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) + } // Do we have a rule-file? if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" { diff --git a/go.mod b/go.mod index f8ddf5709e..c5f84d7d2b 100644 --- a/go.mod +++ b/go.mod @@ -59,9 +59,10 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 - golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 + golang.org/x/mobile v0.0.0-20200123024942-82c397c4c527 // indirect golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect - golang.org/x/sync v0.0.0-20181108010431-42b317875d0f + golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 diff --git a/go.sum b/go.sum index a9a82cae49..d48db47cb3 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,7 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -196,15 +197,34 @@ github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20200123024942-82c397c4c527 h1:470B1IvA7JOANiTULBeEJCIcp53VWOsgckscR836cN8= +golang.org/x/mobile v0.0.0-20200123024942-82c397c4c527/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayERXBdfZjUYoXEf5BTfDfh8= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -213,6 +233,13 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/signer/storage/db_storage.go b/signer/storage/db_storage.go index 2b86f2fd20..0e45ed9da1 100644 --- a/signer/storage/db_storage.go +++ b/signer/storage/db_storage.go @@ -46,6 +46,13 @@ type DBRow struct { val string } +// Default table name for storages +const ( + PasswordTable = "kps" + ConfigTable = "config" + JsTable = "js" +) + // NewDBStorage create new database backed storage func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DBStorage, error) { // sql.Open only validates the input, but didn't create a connection @@ -67,6 +74,9 @@ func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DB // set connection limits db.SetMaxOpenConns(5) + // init table + initTable(driverName, tableName, db) + return &DBStorage{ driverName: driverName, dataSourceName: dataSourceName, @@ -76,6 +86,38 @@ func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DB }, nil } +func initTable(driverName, tableName string, db *sql.DB) error { + var err error + switch driverName { + case "postgres": + _, err = db.Exec(fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + k VARCHAR(255) UNIQUE NOT NULL, + v TEXT NOT NULL +) + `, tableName)) + case "mysql": + _, err = db.Exec(fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s ( + id INT AUTO_INCREMENT PRIMARY KEY, + k VARCHAR(255) UNIQUE NOT NULL, + v TEXT NOT NULL +) + `, tableName)) + case "sqlite3": + _, err = db.Exec(fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s ( + id INTEGER PRIMARY KEY, + k TEXT, + v TEXT +) + `, tableName)) + } + + return err +} + // Put stores a value by key. 0-length keys results in noop. func (s *DBStorage) Put(key, value string) { if len(key) == 0 { @@ -166,10 +208,10 @@ func (s *DBStorage) queryRow(query string, args ...interface{}) (*DBRow, bool, e } var ( - getSQL string = "SELECT * FROM tableName WHERE key = ?" - updateSQL string = "UPDATE tableName SET val = ? WHERE key = ?" - insertSQL string = "INSERT INTO tableName (key, val) VALUES (?, ?)" - deleteSQL string = "DELETE FROM tableName WHERE key = ?" + getSQL string = "SELECT * FROM tableName WHERE k = ?" + updateSQL string = "UPDATE tableName SET v = ? WHERE k = ?" + insertSQL string = "INSERT INTO tableName (k, v) VALUES (?, ?)" + deleteSQL string = "DELETE FROM tableName WHERE k = ?" ) func (s *DBStorage) formatSQL(sql string) string { @@ -179,13 +221,9 @@ func (s *DBStorage) formatSQL(sql string) string { for i := 1; i <= params; i++ { sql = strings.Replace(sql, "?", fmt.Sprintf("$%d", i), 1) } - case "goracle": - params := strings.Count(sql, "?") - for i := 1; i <= params; i++ { - sql = strings.Replace(sql, "?", fmt.Sprintf(":v%d", i), 1) - } default: // for MS SQL Server / MySQL / SQLite + // since they're already using ? as placeholder, do nothing } return strings.ReplaceAll(sql, "tableName", s.tableName) diff --git a/signer/storage/db_storage_test.go b/signer/storage/db_storage_test.go index a2c7dee024..61de22cbc9 100644 --- a/signer/storage/db_storage_test.go +++ b/signer/storage/db_storage_test.go @@ -1,7 +1,6 @@ package storage import ( - "fmt" "io/ioutil" "path/filepath" "testing" @@ -17,9 +16,7 @@ var ( func init() { key = "AES256Key-32Characters1234567890" tmpDir, _ := ioutil.TempDir("", "eth-encrypted-db-storge-test") - fmt.Println(tmpDir) ds, _ = NewDBStorage([]byte(key), "sqlite3", filepath.Join(tmpDir, "test.db"), "kps") - ds.exec("CREATE TABLE IF NOT EXISTS kps (id INTEGER PRIMARY KEY, key TEXT, val TEXT)") } func TestDBStorage(t *testing.T) { @@ -55,10 +52,10 @@ func TestSwappedKeysForDBStorage(t *testing.T) { // now make a modified copy swap := func() { - creds1, _, _ := ds.queryRow("SELECT * FROM kps WHERE key = 'k1'") - creds2, _, _ := ds.queryRow("SELECT * FROM kps WHERE key = 'k2'") - ds.exec("UPDATE kps SET val = ? WHERE key = ?", creds1.val, "k2") - ds.exec("UPDATE kps SET val = ? WHERE key = ?", creds2.val, "k1") + creds1, _, _ := ds.queryRow("SELECT * FROM kps WHERE k = 'k1'") + creds2, _, _ := ds.queryRow("SELECT * FROM kps WHERE k = 'k2'") + ds.exec("UPDATE kps SET v = ? WHERE k = ?", creds1.val, "k2") + ds.exec("UPDATE kps SET v = ? WHERE k = ?", creds2.val, "k1") } swap() if v, _ := ds.Get("k1"); v != "" {