diff --git a/cmd/clef/dbutil/dbutil.go b/cmd/clef/dbutil/dbutil.go index e46c416150..4d5771afd1 100644 --- a/cmd/clef/dbutil/dbutil.go +++ b/cmd/clef/dbutil/dbutil.go @@ -15,13 +15,6 @@ import ( _ "github.com/lib/pq" ) -// Configuration database default table names -const ( - PasswordTable = "kps" - ConfigTable = "config" - JsTable = "js" -) - // Keystore database default table names const ( AccountTable = "accounts" @@ -148,15 +141,15 @@ func (kvstore *KVStore) Get(key string) (string, error) { } // Put stores a value by key. 0-length keys results in noop. -func (kvstore *KVStore) Put(key, value string) { +func (kvstore *KVStore) Put(key, value string) error { if len(key) == 0 { - return + return errors.New("0-length key") } _, err := kvstore.Get(key) if err != nil || err == sql.ErrNoRows { - kvstore.insertRow(key, value) + return kvstore.insertRow(key, value) } else { - kvstore.updateRow(key, value) + return kvstore.updateRow(key, value) } } @@ -199,14 +192,14 @@ func (kvstore *KVStore) All() []string { return result } -func (kvstore *KVStore) insertRow(key, value string) { +func (kvstore *KVStore) insertRow(key, value string) error { sql := kvstore.adjustSQLPlaceholder(insertSQL) - kvstore.exec(sql, key, value) + return kvstore.exec(sql, key, value) } -func (kvstore *KVStore) updateRow(key, value string) { +func (kvstore *KVStore) updateRow(key, value string) error { sql := kvstore.adjustSQLPlaceholder(updateSQL) - kvstore.exec(sql, value, key) + return kvstore.exec(sql, value, key) } func (kvstore *KVStore) adjustSQLPlaceholder(sql string) string { diff --git a/cmd/clef/dbutil/dbutil_test.go b/cmd/clef/dbutil/dbutil_test.go index 869b4e9d11..bf9c0777e9 100644 --- a/cmd/clef/dbutil/dbutil_test.go +++ b/cmd/clef/dbutil/dbutil_test.go @@ -55,7 +55,7 @@ func testPQConfig(t *testing.T, path string) { } func TestKVStoreOperations(t *testing.T) { - kvstore, err := NewKVStore("./dbutil_test_sqlite3.yaml", PasswordTable) + kvstore, err := NewKVStore("./dbutil_test_sqlite3.yaml", "testTable") if err != nil { log.Fatal("Cannot initiate KVStore:", err) } diff --git a/cmd/clef/main.go b/cmd/clef/main.go index e0cc60d342..7a5f6d34fa 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -132,19 +132,11 @@ var ( } keystoreDBFlag = cli.StringFlag{ Name: "keystore-db", - Usage: "General SQL database name which stores keys, currently supported db type are: mysql, postgres", - } - keystoreDBDSNFlag = cli.StringFlag{ - Name: "keystore-db-dsn", - Usage: "Data Source Name (DSN) for keystore-db", + Usage: "Keystore database yaml config file path, currently supported db type are: mysql, postgres", } configDBFlag = cli.StringFlag{ Name: "config-db", - Usage: "General SQL database name which is used to store configuration key value pairs, currently supported db type are: mysql, postgres", - } - configDBDSNFlag = cli.StringFlag{ - Name: "config-db-dsn", - Usage: "Data Source Name (DSN) for config-db", + Usage: "Config database yaml config file path, currently supported db type are: mysql, postgres", } app = cli.NewApp() initCommand = cli.Command{ @@ -236,9 +228,7 @@ func init() { stdiouiFlag, testFlag, keystoreDBFlag, - keystoreDBDSNFlag, configDBFlag, - configDBDSNFlag, advancedMode, } app.Action = signer @@ -320,6 +310,7 @@ You should treat 'masterseed.json' with utmost secrecy and make a backup of it! `) return nil } + func attestFile(ctx *cli.Context) error { if len(ctx.Args()) < 1 { utils.Fatalf("This command requires an argument.") @@ -332,12 +323,8 @@ func attestFile(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) - vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) - confKey := crypto.Keccak256([]byte("config"), stretchedKey) - - // Initialize the encrypted storages - configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confKey) + // Initialize the config storage + configStorage := initConfigStorage(stretchedKey, ctx) val := ctx.Args().First() configStorage.Put("ruleset_sha256", val) log.Info("Ruleset attestation updated", "sha256", val) @@ -363,11 +350,7 @@ func setCredential(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) - vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) - pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) - - pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) + pwStorage := initPasswordStorage(stretchedKey, ctx) pwStorage.Put(address.Hex(), password) log.Info("Credential store updated", "set", address) @@ -391,11 +374,7 @@ func removeCredential(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) - vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) - pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) - - pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) + pwStorage := initPasswordStorage(stretchedKey, ctx) pwStorage.Del(address.Hex()) log.Info("Credential store updated", "unset", address) @@ -446,31 +425,11 @@ func ipcEndpoint(ipcPath, datadir string) string { return ipcPath } -func checkFlag(c *cli.Context) error { - keystoreDB := c.GlobalString(keystoreDBFlag.Name) - keystoreDBDSN := c.GlobalString(keystoreDBDSNFlag.Name) - if keystoreDB != "" && keystoreDBDSN == "" { - return fmt.Errorf("%s has to be set along with %s", keystoreDBDSNFlag.Name, keystoreDBFlag.Name) - } - - configDB := c.GlobalString(configDBFlag.Name) - configDBDSN := c.GlobalString(configDBDSNFlag.Name) - if configDB != "" && configDBDSN == "" { - return fmt.Errorf("%s has to be set along with %s", configDBDSNFlag.Name, configDBFlag.Name) - } - - return nil -} - func signer(c *cli.Context) error { // If we have some unrecognized command, bail out if args := c.Args(); len(args) > 0 { return fmt.Errorf("invalid command: %q", args[0]) } - // check flag legitimacy - if err := checkFlag(c); err != nil { - return err - } if err := initialize(c); err != nil { return err } @@ -495,37 +454,19 @@ func signer(c *cli.Context) error { var ( api core.ExternalAPI - pwStorage storage.Storage = &storage.NoStorage{} - jsStorage storage.Storage = &storage.NoStorage{} - configStorage storage.Storage = &storage.NoStorage{} + pwStorage storage.Storage = storage.NewNoStorage() + jsStorage storage.Storage = storage.NewNoStorage() + configStorage storage.Storage = storage.NewNoStorage() ) configDir := c.GlobalString(configdirFlag.Name) if stretchedKey, err := readMasterKey(c, ui); err != nil { log.Warn("Failed to open master, rules disabled", "err", err) } else { - // Generate domain specific keys - pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) - jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey) - confkey := crypto.Keccak256([]byte("config"), stretchedKey) - - if configDB := c.GlobalString(configDBFlag.Name); configDB != "" { - configDBDSN := c.GlobalString(configDBDSNFlag.Name) - pwStorage, err = storage.NewDBStorage(pwkey, configDB, configDBDSN, storage.PasswordTable) - if err != nil { - utils.Fatalf("Could not connect to config db %v, %v", configDB, configDBDSN) - } - // since we're literally connecting to the same database, so we don't need to check for err for 3 times - jsStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.JsTable) - configStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.ConfigTable) - } else { - vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) - - // Initialize the encrypted storages - pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) - jsStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey) - configStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) - } + // create dedicated storages for different type of configuration + pwStorage = initPasswordStorage(stretchedKey, c) + jsStorage = initJsStorage(stretchedKey, c) + configStorage = initConfigStorage(stretchedKey, c) // Do we have a rule-file? if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" { @@ -1113,3 +1054,40 @@ These data types are defined in the channel between clef and the UI`) fmt.Println(elem) } } + +// Constants for file/db storage vault/table name +const ( + PasswordStorageVault string = "credentials" + JsStorageVault string = "jsstorage" + ConfigStorageVault string = "config" +) + +func initPasswordStorage(stretchedKey []byte, c *cli.Context) storage.Storage { + return initStorage(PasswordStorageVault, stretchedKey, c) +} + +func initJsStorage(stretchedKey []byte, c *cli.Context) storage.Storage { + return initStorage(JsStorageVault, stretchedKey, c) +} + +func initConfigStorage(stretchedKey []byte, c *cli.Context) storage.Storage { + return initStorage(ConfigStorageVault, stretchedKey, c) +} + +func initStorage(storageName string, stretchedKey []byte, c *cli.Context) storage.Storage { + configDir := c.GlobalString(configdirFlag.Name) + key := crypto.Keccak256([]byte(storageName), stretchedKey) + + if configDB := c.GlobalString(configDBFlag.Name); configDB != "" { + // if config-db is specified, create db backed storage + storage, err := storage.NewDBStorage(configDB, storageName, key) + if err != nil { + utils.Fatalf("Cannot init database storage for %s", storageName) + } + return storage + } else { + // otherwise create file storage + vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) + return storage.NewFileStorage(filepath.Join(vaultLocation, storageName+".json"), key) + } +} diff --git a/go.mod b/go.mod index c5f84d7d2b..0f60228115 100644 --- a/go.mod +++ b/go.mod @@ -70,5 +70,6 @@ require ( gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/urfave/cli.v1 v1.20.0 - gotest.tools v2.2.0+incompatible // indirect + gopkg.in/yaml.v2 v2.2.2 + gotest.tools v2.2.0+incompatible ) diff --git a/signer/core/api_test.go b/signer/core/api_test.go index 48d0c4f967..160f0e39b1 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -126,9 +126,8 @@ func setup(ksLoc string, t *testing.T) (*core.SignerAPI, *headlessUi) { } ui := &headlessUi{make(chan string, 20), make(chan string, 20)} am := core.StartClefAccountManager(ksLoc, true, true, "") - api := core.NewSignerAPI(am, 1337, true, ui, db, true, &storage.NoStorage{}) + api := core.NewSignerAPI(am, 1337, true, ui, db, true, storage.NewNoStorage()) return api, ui - } func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { diff --git a/signer/storage/db_storage.go b/signer/storage/db_storage.go index 7662e3f074..0512e1ca53 100644 --- a/signer/storage/db_storage.go +++ b/signer/storage/db_storage.go @@ -32,8 +32,8 @@ func (api *DBStorageAPI) Get(key string) (string, error) { } // Put stores a value by key. 0-length keys results in noop. -func (api *DBStorageAPI) Put(key, value string) { - api.kvstore.Put(key, value) +func (api *DBStorageAPI) Put(key, value string) error { + return api.kvstore.Put(key, value) } // Del removes a key-value pair. If the key doesn't exist, the method is a noop. diff --git a/signer/storage/ephemeral_storage.go b/signer/storage/ephemeral_storage.go index a237f23c39..3ffb25c7d9 100644 --- a/signer/storage/ephemeral_storage.go +++ b/signer/storage/ephemeral_storage.go @@ -7,11 +7,12 @@ type EphemeralStorageAPI struct { } // Put stores a value by key. 0-length keys results in noop. -func (s *EphemeralStorageAPI) Put(key, value string) { +func (s *EphemeralStorageAPI) Put(key, value string) error { if len(key) == 0 { - return + return ErrZeroKey } s.data[key] = value + return nil } // Get returns the previously stored value, or an error if the key is 0-length diff --git a/signer/storage/file_storage.go b/signer/storage/file_storage.go index 84d131a2cb..6345a2b52a 100644 --- a/signer/storage/file_storage.go +++ b/signer/storage/file_storage.go @@ -31,19 +31,21 @@ type FileStorageAPI struct { } // Put stores a value by key. 0-length keys results in noop. -func (s *FileStorageAPI) Put(key, value string) { +func (s *FileStorageAPI) Put(key, value string) error { if len(key) == 0 { - return + return ErrZeroKey } data, err := s.readStorage() if err != nil { log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename) - return + return err } data[key] = value if err = s.writeStorage(data); err != nil { log.Warn("Failed to write entry", "err", err) + return err } + return nil } // Get returns the previously stored value, or an error if it does not exist or diff --git a/signer/storage/no_storage.go b/signer/storage/no_storage.go index 3f213ef892..2a3e943dab 100644 --- a/signer/storage/no_storage.go +++ b/signer/storage/no_storage.go @@ -6,7 +6,9 @@ import "errors" type NoStorageAPI struct{} // Put is a dummy function that do nothing -func (s *NoStorageAPI) Put(key, value string) {} +func (s *NoStorageAPI) Put(key, value string) error { + return errors.New("I don't know how to remember") +} // Del is a dummy function that do nothing func (s *NoStorageAPI) Del(key string) {} diff --git a/signer/storage/storage.go b/signer/storage/storage.go index 219b2cfa44..b451b2a6a8 100644 --- a/signer/storage/storage.go +++ b/signer/storage/storage.go @@ -35,7 +35,7 @@ var ( // storageAPI is the interface that defines interactions with backend storage client type storageAPI interface { // Put stores a value by key. 0-length keys results in noop. - Put(key, value string) + Put(key, value string) error // Get returns the previously stored value, or an error if the key is 0-length // or unknown. @@ -80,24 +80,24 @@ func (s *Storage) Get(key string) (string, error) { // Put encrypts the value field with key as additionalData to prevent value swap attack. // Then calls the underlying storageApi's Put function to persist the key/value pair -func (s *Storage) Put(key, value string) { +func (s *Storage) Put(key, value string) error { if len(key) == 0 { - return + return ErrZeroKey } ciphertext, iv, err := Encrypt(s.key, []byte(value), []byte(key)) if err != nil { log.Warn("Failed to encrypt entry", "err", err) - return + return err } encrypted := StoredCredential{Iv: iv, CipherText: ciphertext} raw, err := json.Marshal(encrypted) if err != nil { log.Warn("Failed to marshal credential", "err", err) - return + return err } - s.api.Put(key, string(raw)) + return s.api.Put(key, string(raw)) } // Del calls the underlying storageApi's Del function to delete the key/value pair @@ -127,26 +127,26 @@ func NewNoStorage() Storage { } // NewDBStorage creates a database storage -func NewDBStorage(path, table string, key []byte) (*Storage, error) { +func NewDBStorage(path, table string, key []byte) (Storage, error) { kvstore, err := dbutil.NewKVStore(path, table) if err != nil { - return nil, err + return NewNoStorage(), err } api := &DBStorageAPI{ kvstore: kvstore, } - return &Storage{ + return Storage{ api: api, key: key, }, nil } // NewFileStorage creates a storage type which is backed by a json-file. -func NewFileStorage(filename string, key []byte) *Storage { +func NewFileStorage(filename string, key []byte) Storage { api := &FileStorageAPI{ filename: filename, } - return &Storage{ + return Storage{ api: api, key: key, }