signer/core,storage, cmd/clef/dbutil finish rework storages for clef

cmd/clef/dbutil/dbutil.go

@@ -15,13 +15,6 @@ import (
 	_ "github.com/lib/pq"
 )
 
-// Configuration database default table names
-const (
-	PasswordTable = "kps"
-	ConfigTable   = "config"
-	JsTable       = "js"
-)
-
 // Keystore database default table names
 const (
 	AccountTable = "accounts"
@@ -148,15 +141,15 @@ func (kvstore *KVStore) Get(key string) (string, error) {
 }
 
 // Put stores a value by key. 0-length keys results in noop.
-func (kvstore *KVStore) Put(key, value string) {
+func (kvstore *KVStore) Put(key, value string) error {
 	if len(key) == 0 {
-		return
+		return errors.New("0-length key")
 	}
 	_, err := kvstore.Get(key)
 	if err != nil || err == sql.ErrNoRows {
-		kvstore.insertRow(key, value)
+		return kvstore.insertRow(key, value)
 	} else {
-		kvstore.updateRow(key, value)
+		return kvstore.updateRow(key, value)
 	}
 }
 
@@ -199,14 +192,14 @@ func (kvstore *KVStore) All() []string {
 	return result
 }
 
-func (kvstore *KVStore) insertRow(key, value string) {
+func (kvstore *KVStore) insertRow(key, value string) error {
 	sql := kvstore.adjustSQLPlaceholder(insertSQL)
-	kvstore.exec(sql, key, value)
+	return kvstore.exec(sql, key, value)
 }
 
-func (kvstore *KVStore) updateRow(key, value string) {
+func (kvstore *KVStore) updateRow(key, value string) error {
 	sql := kvstore.adjustSQLPlaceholder(updateSQL)
-	kvstore.exec(sql, value, key)
+	return kvstore.exec(sql, value, key)
 }
 
 func (kvstore *KVStore) adjustSQLPlaceholder(sql string) string {

cmd/clef/dbutil/dbutil_test.go

@@ -55,7 +55,7 @@ func testPQConfig(t *testing.T, path string) {
 }
 
 func TestKVStoreOperations(t *testing.T) {
-	kvstore, err := NewKVStore("./dbutil_test_sqlite3.yaml", PasswordTable)
+	kvstore, err := NewKVStore("./dbutil_test_sqlite3.yaml", "testTable")
 	if err != nil {
 		log.Fatal("Cannot initiate KVStore:", err)
 	}

cmd/clef/main.go

@@ -132,19 +132,11 @@ var (
 	}
 	keystoreDBFlag = cli.StringFlag{
 		Name:  "keystore-db",
-		Usage: "General SQL database name which stores keys, currently supported db type are: mysql, postgres",
-	}
-	keystoreDBDSNFlag = cli.StringFlag{
-		Name:  "keystore-db-dsn",
-		Usage: "Data Source Name (DSN) for keystore-db",
+		Usage: "Keystore database yaml config file path, currently supported db type are: mysql, postgres",
 	}
 	configDBFlag = cli.StringFlag{
 		Name:  "config-db",
-		Usage: "General SQL database name which is used to store configuration key value pairs, currently supported db type are: mysql, postgres",
-	}
-	configDBDSNFlag = cli.StringFlag{
-		Name:  "config-db-dsn",
-		Usage: "Data Source Name (DSN) for config-db",
+		Usage: "Config database yaml config file path, currently supported db type are: mysql, postgres",
 	}
 	app         = cli.NewApp()
 	initCommand = cli.Command{
@@ -236,9 +228,7 @@ func init() {
 		stdiouiFlag,
 		testFlag,
 		keystoreDBFlag,
-		keystoreDBDSNFlag,
 		configDBFlag,
-		configDBDSNFlag,
 		advancedMode,
 	}
 	app.Action = signer
@@ -320,6 +310,7 @@ You should treat 'masterseed.json' with utmost secrecy and make a backup of it!
 `)
 	return nil
 }
+
 func attestFile(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
 		utils.Fatalf("This command requires an argument.")
@@ -332,12 +323,8 @@ func attestFile(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.GlobalString(configdirFlag.Name)
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
-	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
-
-	// Initialize the encrypted storages
-	configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confKey)
+	// Initialize the config storage
+	configStorage := initConfigStorage(stretchedKey, ctx)
 	val := ctx.Args().First()
 	configStorage.Put("ruleset_sha256", val)
 	log.Info("Ruleset attestation updated", "sha256", val)
@@ -363,11 +350,7 @@ func setCredential(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.GlobalString(configdirFlag.Name)
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
-	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
-
-	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	pwStorage := initPasswordStorage(stretchedKey, ctx)
 	pwStorage.Put(address.Hex(), password)
 
 	log.Info("Credential store updated", "set", address)
@@ -391,11 +374,7 @@ func removeCredential(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.GlobalString(configdirFlag.Name)
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
-	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
-
-	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	pwStorage := initPasswordStorage(stretchedKey, ctx)
 	pwStorage.Del(address.Hex())
 
 	log.Info("Credential store updated", "unset", address)
@@ -446,31 +425,11 @@ func ipcEndpoint(ipcPath, datadir string) string {
 	return ipcPath
 }
 
-func checkFlag(c *cli.Context) error {
-	keystoreDB := c.GlobalString(keystoreDBFlag.Name)
-	keystoreDBDSN := c.GlobalString(keystoreDBDSNFlag.Name)
-	if keystoreDB != "" && keystoreDBDSN == "" {
-		return fmt.Errorf("%s has to be set along with %s", keystoreDBDSNFlag.Name, keystoreDBFlag.Name)
-	}
-
-	configDB := c.GlobalString(configDBFlag.Name)
-	configDBDSN := c.GlobalString(configDBDSNFlag.Name)
-	if configDB != "" && configDBDSN == "" {
-		return fmt.Errorf("%s has to be set along with %s", configDBDSNFlag.Name, configDBFlag.Name)
-	}
-
-	return nil
-}
-
 func signer(c *cli.Context) error {
 	// If we have some unrecognized command, bail out
 	if args := c.Args(); len(args) > 0 {
 		return fmt.Errorf("invalid command: %q", args[0])
 	}
-	// check flag legitimacy
-	if err := checkFlag(c); err != nil {
-		return err
-	}
 	if err := initialize(c); err != nil {
 		return err
 	}
@@ -495,37 +454,19 @@ func signer(c *cli.Context) error {
 
 	var (
 		api           core.ExternalAPI
-		pwStorage     storage.Storage = &storage.NoStorage{}
-		jsStorage     storage.Storage = &storage.NoStorage{}
-		configStorage storage.Storage = &storage.NoStorage{}
+		pwStorage     storage.Storage = storage.NewNoStorage()
+		jsStorage     storage.Storage = storage.NewNoStorage()
+		configStorage storage.Storage = storage.NewNoStorage()
 	)
 
 	configDir := c.GlobalString(configdirFlag.Name)
 	if stretchedKey, err := readMasterKey(c, ui); err != nil {
 		log.Warn("Failed to open master, rules disabled", "err", err)
 	} else {
-		// Generate domain specific keys
-		pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
-		jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey)
-		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
-
-		if configDB := c.GlobalString(configDBFlag.Name); configDB != "" {
-			configDBDSN := c.GlobalString(configDBDSNFlag.Name)
-			pwStorage, err = storage.NewDBStorage(pwkey, configDB, configDBDSN, storage.PasswordTable)
-			if err != nil {
-				utils.Fatalf("Could not connect to config db %v, %v", configDB, configDBDSN)
-			}
-			// since we're literally connecting to the same database, so we don't need to check for err for 3 times
-			jsStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.JsTable)
-			configStorage, _ = storage.NewDBStorage(jskey, configDB, configDBDSN, storage.ConfigTable)
-		} else {
-			vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
-
-			// Initialize the encrypted storages
-			pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-			jsStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
-			configStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
-		}
+		// create dedicated storages for different type of configuration
+		pwStorage = initPasswordStorage(stretchedKey, c)
+		jsStorage = initJsStorage(stretchedKey, c)
+		configStorage = initConfigStorage(stretchedKey, c)
 
 		// Do we have a rule-file?
 		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {
@@ -1113,3 +1054,40 @@ These data types are defined in the channel between clef and the UI`)
 		fmt.Println(elem)
 	}
 }
+
+// Constants for file/db storage vault/table name
+const (
+	PasswordStorageVault string = "credentials"
+	JsStorageVault       string = "jsstorage"
+	ConfigStorageVault   string = "config"
+)
+
+func initPasswordStorage(stretchedKey []byte, c *cli.Context) storage.Storage {
+	return initStorage(PasswordStorageVault, stretchedKey, c)
+}
+
+func initJsStorage(stretchedKey []byte, c *cli.Context) storage.Storage {
+	return initStorage(JsStorageVault, stretchedKey, c)
+}
+
+func initConfigStorage(stretchedKey []byte, c *cli.Context) storage.Storage {
+	return initStorage(ConfigStorageVault, stretchedKey, c)
+}
+
+func initStorage(storageName string, stretchedKey []byte, c *cli.Context) storage.Storage {
+	configDir := c.GlobalString(configdirFlag.Name)
+	key := crypto.Keccak256([]byte(storageName), stretchedKey)
+
+	if configDB := c.GlobalString(configDBFlag.Name); configDB != "" {
+		// if config-db is specified, create db backed storage
+		storage, err := storage.NewDBStorage(configDB, storageName, key)
+		if err != nil {
+			utils.Fatalf("Cannot init database storage for %s", storageName)
+		}
+		return storage
+	} else {
+		// otherwise create file storage
+		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+		return storage.NewFileStorage(filepath.Join(vaultLocation, storageName+".json"), key)
+	}
+}

go.mod

@@ -70,5 +70,6 @@ require (
 	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0
-	gotest.tools v2.2.0+incompatible // indirect
+	gopkg.in/yaml.v2 v2.2.2
+	gotest.tools v2.2.0+incompatible
 )

signer/core/api_test.go

@@ -126,9 +126,8 @@ func setup(ksLoc string, t *testing.T) (*core.SignerAPI, *headlessUi) {
 	}
 	ui := &headlessUi{make(chan string, 20), make(chan string, 20)}
 	am := core.StartClefAccountManager(ksLoc, true, true, "")
-	api := core.NewSignerAPI(am, 1337, true, ui, db, true, &storage.NoStorage{})
+	api := core.NewSignerAPI(am, 1337, true, ui, db, true, storage.NewNoStorage())
 	return api, ui
-
 }
 
 func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) {

signer/storage/db_storage.go

@@ -32,8 +32,8 @@ func (api *DBStorageAPI) Get(key string) (string, error) {
 }
 
 // Put stores a value by key. 0-length keys results in noop.
-func (api *DBStorageAPI) Put(key, value string) {
-	api.kvstore.Put(key, value)
+func (api *DBStorageAPI) Put(key, value string) error {
+	return api.kvstore.Put(key, value)
 }
 
 // Del removes a key-value pair. If the key doesn't exist, the method is a noop.

signer/storage/ephemeral_storage.go

@@ -7,11 +7,12 @@ type EphemeralStorageAPI struct {
 }
 
 // Put stores a value by key. 0-length keys results in noop.
-func (s *EphemeralStorageAPI) Put(key, value string) {
+func (s *EphemeralStorageAPI) Put(key, value string) error {
 	if len(key) == 0 {
-		return
+		return ErrZeroKey
 	}
 	s.data[key] = value
+	return nil
 }
 
 // Get returns the previously stored value, or an error if the key is 0-length

signer/storage/file_storage.go

@@ -31,19 +31,21 @@ type FileStorageAPI struct {
 }
 
 // Put stores a value by key. 0-length keys results in noop.
-func (s *FileStorageAPI) Put(key, value string) {
+func (s *FileStorageAPI) Put(key, value string) error {
 	if len(key) == 0 {
-		return
+		return ErrZeroKey
 	}
 	data, err := s.readStorage()
 	if err != nil {
 		log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
-		return
+		return err
 	}
 	data[key] = value
 	if err = s.writeStorage(data); err != nil {
 		log.Warn("Failed to write entry", "err", err)
+		return err
 	}
+	return nil
 }
 
 // Get returns the previously stored value, or an error if it does not exist or

signer/storage/no_storage.go

@@ -6,7 +6,9 @@ import "errors"
 type NoStorageAPI struct{}
 
 // Put is a dummy function that do nothing
-func (s *NoStorageAPI) Put(key, value string) {}
+func (s *NoStorageAPI) Put(key, value string) error {
+	return errors.New("I don't know how to remember")
+}
 
 // Del is a dummy function that do nothing
 func (s *NoStorageAPI) Del(key string) {}

signer/storage/storage.go

@@ -35,7 +35,7 @@ var (
 // storageAPI is the interface that defines interactions with backend storage client
 type storageAPI interface {
 	// Put stores a value by key. 0-length keys results in noop.
-	Put(key, value string)
+	Put(key, value string) error
 
 	// Get returns the previously stored value, or an error if the key is 0-length
 	// or unknown.
@@ -80,24 +80,24 @@ func (s *Storage) Get(key string) (string, error) {
 
 // Put encrypts the value field with key as additionalData to prevent value swap attack.
 // Then calls the underlying storageApi's Put function to persist the key/value pair
-func (s *Storage) Put(key, value string) {
+func (s *Storage) Put(key, value string) error {
 	if len(key) == 0 {
-		return
+		return ErrZeroKey
 	}
 
 	ciphertext, iv, err := Encrypt(s.key, []byte(value), []byte(key))
 	if err != nil {
 		log.Warn("Failed to encrypt entry", "err", err)
-		return
+		return err
 	}
 
 	encrypted := StoredCredential{Iv: iv, CipherText: ciphertext}
 	raw, err := json.Marshal(encrypted)
 	if err != nil {
 		log.Warn("Failed to marshal credential", "err", err)
-		return
+		return err
 	}
-	s.api.Put(key, string(raw))
+	return s.api.Put(key, string(raw))
 }
 
 // Del calls the underlying storageApi's Del function to delete the key/value pair
@@ -127,26 +127,26 @@ func NewNoStorage() Storage {
 }
 
 // NewDBStorage creates a database storage
-func NewDBStorage(path, table string, key []byte) (*Storage, error) {
+func NewDBStorage(path, table string, key []byte) (Storage, error) {
 	kvstore, err := dbutil.NewKVStore(path, table)
 	if err != nil {
-		return nil, err
+		return NewNoStorage(), err
 	}
 	api := &DBStorageAPI{
 		kvstore: kvstore,
 	}
-	return &Storage{
+	return Storage{
 		api: api,
 		key: key,
 	}, nil
 }
 
 // NewFileStorage creates a storage type which is backed by a json-file.
-func NewFileStorage(filename string, key []byte) *Storage {
+func NewFileStorage(filename string, key []byte) Storage {
 	api := &FileStorageAPI{
 		filename: filename,
 	}
-	return &Storage{
+	return Storage{
 		api: api,
 		key: key,
 	}