From 56dd351ac6352eef8fb0352ea17aa6d2c98f9aa8 Mon Sep 17 00:00:00 2001 From: priom Date: Wed, 16 May 2018 15:48:16 -0400 Subject: [PATCH 1/9] initial miner reward --- shyftDb/postgres_setup/create_tables.psql | 5 + shyftDb/postgres_setup/drop_tables.psql | 3 +- shyftDb/shyft_database_util.go | 806 ++++++++++++++++++++++ shyftFullReset.sh | 0 shyftdb/database.go | 383 ---------- shyftdb/interface.go | 31 - shyftdb/shyft_database_util.go | 56 +- simulations/sendTransactions.js | 36 +- 8 files changed, 870 insertions(+), 450 deletions(-) create mode 100644 shyftDb/shyft_database_util.go create mode 100644 shyftFullReset.sh delete mode 100644 shyftdb/database.go delete mode 100644 shyftdb/interface.go diff --git a/shyftDb/postgres_setup/create_tables.psql b/shyftDb/postgres_setup/create_tables.psql index 80afba9790..c24638c450 100644 --- a/shyftDb/postgres_setup/create_tables.psql +++ b/shyftDb/postgres_setup/create_tables.psql @@ -37,4 +37,9 @@ CREATE TABLE IF NOT EXISTS accounts ( CREATE TABLE IF NOT EXISTS contracts ( txHash text +); + +CREATE TABLE IF NOT EXISTS mined_blocks ( + blockNumber bigint, + addr text ); \ No newline at end of file diff --git a/shyftDb/postgres_setup/drop_tables.psql b/shyftDb/postgres_setup/drop_tables.psql index 5df13ed64c..2921358098 100644 --- a/shyftDb/postgres_setup/drop_tables.psql +++ b/shyftDb/postgres_setup/drop_tables.psql @@ -1,4 +1,5 @@ DROP TABLE txs; DROP TABLE blocks; DROP TABLE accounts; -DROP TABLE contracts; \ No newline at end of file +DROP TABLE contracts; +DROP TABLE mined_blocks; \ No newline at end of file diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go new file mode 100644 index 0000000000..c821f50423 --- /dev/null +++ b/shyftDb/shyft_database_util.go @@ -0,0 +1,806 @@ +package shyftdb + +import ( + "encoding/json" + "fmt" + "math/big" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "time" + "strconv" + "database/sql" + "log" + + _ "github.com/lib/pq" + "reflect" +) + +//SBlock type +type SBlock struct { + Hash string + Coinbase string + Number string + GasUsed string + GasLimit string + TxCount string + UncleCount string + Age string + ParentHash string + UncleHash string + Difficulty string + Size string + Nonce string +} + +//blockRes struct +type blockRes struct { + hash string + coinbase string + number string + Blocks []SBlock +} + +type SAccounts struct { + Addr string + Balance string + TxCountAccount string +} + +type accountRes struct { + addr string + balance string + AllAccounts []SAccounts +} + +//ShyftTxEntry structure +type ShyftTxEntry struct { + TxHash common.Hash + To *common.Address + From *common.Address + BlockHash string + Amount *big.Int + GasPrice *big.Int + Gas uint64 + Cost *big.Int + Nonce uint64 + Data []byte +} + +type txRes struct { + TxEntry []ShyftTxEntryPretty +} + +type ShyftTxEntryPretty struct { + TxHash string + To string + From string + BlockHash string + BlockNumber string + Amount uint64 + GasPrice uint64 + Gas uint64 + Cost uint64 + Nonce uint64 + Data []byte +} + +type ShyftAccountEntry struct { + Balance string + Txs []string +} + +type SendAndReceive struct { + To string + From string + Amount string + Address string + Balance string + TxCountAccount string +} + +//WriteBlock writes to block info to sql db +func WriteBlock(sqldb *sql.DB, block *types.Block, receipts []*types.Receipt) error { + //Need to create field in postgres db isContract : True || False + //Need to fix nonce numeric issue (attempt to reproduce and record) + //Need to update tx To Field where null with Contract Address + //Need to update account table with that Contract Address + //Need to update AccountNonce and Balance of Tx To field || Contract Address + WriteMinerRewards(sqldb,block) + coinbase := block.Header().Coinbase.String() + number := block.Header().Number.String() + gasUsed := block.Header().GasUsed + gasLimit := block.Header().GasLimit + txCount := block.Transactions().Len() + uncleCount := len(block.Uncles()) + parentHash := block.ParentHash().String() + uncleHash := block.UncleHash().String() + blockDifficulty := block.Difficulty().String() + blockSize := block.Size().String() + blockNonce := block.Nonce() + + // Convert the receipts into their storage form and serialize them + storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) + for i, receipt := range receipts { + storageReceipts[i] = (*types.ReceiptForStorage)(receipt) + var txHashFromReciept = (*types.ReceiptForStorage)(receipt).TxHash + var statusFromReciept = (*types.ReceiptForStorage)(receipt).Status + var contractAddressFromReciept = (*types.ReceiptForStorage)(receipt).ContractAddress + if statusFromReciept == 1 { + fmt.Println("THIS IS statusFromReciept", "SUCCESS", statusFromReciept) + } + if statusFromReciept == 0 { + fmt.Println("THIS IS statusFromReciept", "FAIL", statusFromReciept) + } + + if block.Transactions()[0].To() == nil { + updateSQLStatement := `UPDATE txs SET to_addr = ($2) WHERE txHash = ($1)` + _, error := sqldb.Exec(updateSQLStatement, txHashFromReciept.String(), contractAddressFromReciept.String()) + if error != nil { + panic(error) + } + } + + + fmt.Println("THIS IS txHashFromReciept", txHashFromReciept.String()) + fmt.Println("THIS IS contractAddressFromReciept", contractAddressFromReciept.String()) + } + + i, err := strconv.ParseInt(block.Time().String(), 10, 64) + if err != nil { + panic(err) + } + age := time.Unix(i, 0) + + sqlStatement := `INSERT INTO blocks(hash, coinbase, number, gasUsed, gasLimit, txCount, uncleCount, age, parentHash, uncleHash, difficulty, size, nonce) VALUES(($1), ($2), ($3), ($4), ($5), ($6), ($7), ($8), ($9), ($10), ($11), ($12),($13)) RETURNING number` + qerr := sqldb.QueryRow(sqlStatement, block.Header().Hash().Hex(), coinbase, number, gasUsed, gasLimit, txCount, uncleCount, age, parentHash, uncleHash, blockDifficulty, blockSize, blockNonce).Scan(&number) + if qerr != nil { + panic(qerr) + } + + if block.Transactions().Len() > 0 { + for _, tx := range block.Transactions() { + //WriteMinerRewards(sqldb, block) + WriteTransactions(sqldb, tx, block.Header().Hash(), block.Header().Number.String()) + if block.Transactions()[0].To() != nil { + WriteFromBalance(sqldb, tx) + } + if block.Transactions()[0].To() == nil { + WriteContractBalance(sqldb, tx) + WriteContractsTxHashReferences(sqldb, tx) + } + } + } + return nil +} + +//WriteTransactions writes to sqldb +func WriteTransactions(sqldb *sql.DB, tx *types.Transaction, blockHash common.Hash, blockNumber string) error { + txData := ShyftTxEntry{ + TxHash: tx.Hash(), + From: tx.From(), + To: tx.To(), + BlockHash: blockHash.Hex(), + Amount: tx.Value(), + Cost: tx.Cost(), + GasPrice: tx.GasPrice(), + Gas: tx.Gas(), + Nonce: tx.Nonce(), + Data: tx.Data(), + } + + txHash := txData.TxHash.Hex() + from := txData.From.Hex() + blockHasher := txData.BlockHash + amount := txData.Amount.String() + gasPrice := txData.GasPrice.String() + txFee := txData.Cost.String() + nonce := txData.Nonce + gas := txData.Gas + data := txData.Data + to := txData.To + var isContract bool + if (to == nil){ + var retNonce string + isContract = true + sqlStatement := `INSERT INTO txs(txhash, from_addr, blockhash, blockNumber, amount, gasprice, gas, txfee, nonce, isContract, data) VALUES(($1), ($2), ($3), ($4), ($5), ($6), ($7), ($8), ($9), ($10), ($11)) RETURNING nonce` + qerr := sqldb.QueryRow(sqlStatement, txHash, from, blockHasher, blockNumber, amount, gasPrice, gas, txFee, nonce, isContract, data).Scan(&retNonce) + + if qerr != nil { + panic(qerr) + } + } else { + var retNonce string + isContract = false + sqlStatement := `INSERT INTO txs(txhash, from_addr, to_addr, blockhash, blockNumber, amount, gasprice, gas, txfee, nonce, isContract, data) VALUES(($1), ($2), ($3), ($4), ($5), ($6), ($7), ($8), ($9), ($10), ($11), ($12)) RETURNING nonce` + qerr := sqldb.QueryRow(sqlStatement, txHash, from, to.Hex(), blockHasher, blockNumber, amount, gasPrice, gas, txFee, nonce, isContract, data).Scan(&retNonce) + + if qerr != nil { + panic(qerr) + } + } + + return nil +} + +func WriteContractsTxHashReferences(sqldb *sql.DB, tx *types.Transaction) error { + txHash := tx.Hash().Hex() + + sqlStatement := `INSERT INTO contracts(txHash) VALUES(($1)) RETURNING txHash` + insertErr := sqldb.QueryRow(sqlStatement, txHash).Scan(&txHash) + if insertErr != nil { + panic(insertErr) + } + return nil +} + +func WriteContractBalance(sqldb *sql.DB, tx *types.Transaction) error { + sendAndReceiveData,balanceSen,accountNonceSen := WriteContractBalanceHelper(sqldb, tx) + fromAddr := sendAndReceiveData.From + amount := sendAndReceiveData.Amount + balanceSender := balanceSen + + var response string + sqlExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(sqlExistsStatement, fromAddr).Scan(&response) + switch { + case err == sql.ErrNoRows: + fmt.Println("NO ROWS RAN") + //i, err := strconv.Atoi(accountNonceSen) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println("accountnonce", i) + //fmt.Println(reflect.TypeOf(i)) + sqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + insertErr := sqldb.QueryRow(sqlStatement, fromAddr, amount, accountNonceSen).Scan(&fromAddr) + if insertErr != nil { + panic(insertErr) + } + case err != nil: + log.Fatal(err) + default: + var newBalanceSender big.Int + var newAccountNonceSender big.Int + var nonceIncrement = big.NewInt(1) + updateSQLStatement := `UPDATE accounts SET balance = ($2), txCountAccount = ($3) WHERE addr = ($1)` + + s := new(big.Int) + _, error := fmt.Sscan(balanceSender, s) + if error != nil { + log.Println("error scanning value:", error) + } + + accountS := new(big.Int) + _, errors := fmt.Sscan(accountNonceSen, accountS) + if errors != nil { + log.Println("error scanning value:", error) + } + + newBalanceSender.Sub(s, tx.Value()) + newAccountNonceSender.Add(accountS, nonceIncrement) + + _, err = sqldb.Exec(updateSQLStatement, fromAddr, newBalanceSender.String(), newAccountNonceSender.String()) + if err != nil { + panic(err) + } + } + return nil +} + +func WriteContractBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, string, string) { + sendAndReceiveData := SendAndReceive{ + From: tx.From().Hex(), + Amount: tx.Value().String(), + } + + fromAddr := sendAndReceiveData.From + getAccountBalanceSender:= GetAccount(sqldb, fromAddr) + + var senderBalance SendAndReceive + if err := json.Unmarshal([]byte(getAccountBalanceSender), &senderBalance); err != nil { + log.Fatal(err) + } + balanceSender := senderBalance.Balance + accountNonceSender := senderBalance.TxCountAccount + + return sendAndReceiveData, balanceSender, accountNonceSender +} + +//WriteFromBalance writes senders balance to accounts db +func WriteFromBalance(sqldb *sql.DB, tx *types.Transaction) error { + //IF to address is nil (which means its contract creation) + //Need to create a condition (flag) check and then change how the nonce increment works + sendAndReceiveData, balanceRec, balanceSen, accountNonceRec, accountNonceSen := WriteBalanceHelper(sqldb, tx) + toAddr := sendAndReceiveData.To + fromAddr := sendAndReceiveData.From + amount := sendAndReceiveData.Amount + balanceReceiver := balanceRec + balanceSender := balanceSen + + var response string + sqlExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(sqlExistsStatement, toAddr).Scan(&response) + switch { + case err == sql.ErrNoRows: + fmt.Println("NO ROWS RAN") + i, err := strconv.Atoi(accountNonceRec) + if err != nil { + fmt.Println(err) + } + //fmt.Println("accountnonce", i) + //fmt.Println(reflect.TypeOf(i)) + sqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + insertErr := sqldb.QueryRow(sqlStatement, toAddr, amount, i).Scan(&toAddr) + if insertErr != nil { + panic(insertErr) + } + case err != nil: + log.Fatal(err) + default: + var newBalanceReceiver big.Int + var newBalanceSender big.Int + var newAccountNonceReceiver big.Int + var newAccountNonceSender big.Int + var nonceIncrement = big.NewInt(1) + updateSQLStatement := `UPDATE accounts SET balance = ($2), txCountAccount = ($3) WHERE addr = ($1)` + + r := new(big.Int) + _, err := fmt.Sscan(balanceReceiver, r) + if err != nil { + log.Println("error scanning value:", err) + } + + s := new(big.Int) + _, error := fmt.Sscan(balanceSender, s) + if error != nil { + log.Println("error scanning value:", error) + } + + accountR := new(big.Int) + _, er := fmt.Sscan(accountNonceRec, accountR) + if er != nil { + log.Println("error scanning value:", er) + } + + accountS := new(big.Int) + _, errors := fmt.Sscan(accountNonceSen, accountS) + if errors != nil { + log.Println("error scanning value:", error) + } + + newBalanceReceiver.Add(r, tx.Value()) + newBalanceSender.Sub(s, tx.Value()) + + newAccountNonceReceiver.Add(accountR, nonceIncrement) + newAccountNonceSender.Add(accountS, nonceIncrement) + + _, err = sqldb.Exec(updateSQLStatement, toAddr, newBalanceReceiver.String(), newAccountNonceReceiver.String()) + if err != nil { + panic(err) + } + + _, err = sqldb.Exec(updateSQLStatement, fromAddr, newBalanceSender.String(), newAccountNonceSender.String()) + if err != nil { + panic(err) + } + } + return nil +} + +func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, string, string, string, string) { + sendAndReceiveData := SendAndReceive{ + To: tx.To().Hex(), + From: tx.From().Hex(), + Amount: tx.Value().String(), + } + + toAddr := sendAndReceiveData.To + fromAddr := sendAndReceiveData.From + + getAccountBalanceReceiver := GetAccount(sqldb, toAddr) + getAccountBalanceSender:= GetAccount(sqldb, fromAddr) + + var receiverBalance SendAndReceive + if err := json.Unmarshal([]byte(getAccountBalanceReceiver), &receiverBalance); err != nil { + log.Fatal(err) + } + + var senderBalance SendAndReceive + if err := json.Unmarshal([]byte(getAccountBalanceSender), &senderBalance); err != nil { + log.Fatal(err) + } + + balanceReceiver := receiverBalance.Balance + balanceSender := senderBalance.Balance + + accountNonceReceiver := receiverBalance.TxCountAccount + accountNonceSender := senderBalance.TxCountAccount + + return sendAndReceiveData, balanceReceiver, balanceSender, accountNonceReceiver, accountNonceSender +} + +func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { + minerAddr := block.Coinbase().String() + fmt.Println("\n\n\t\t", minerAddr, "\n\n") + // Calculate the total gas used in the block + var totalGas big.Int + for _, tx := range block.Transactions() { + totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) + } + + //TODO: Calculate Block Reward + // totalReward := totalGas.Add(&totalGas, MINER_REWARD) + totalReward := totalGas + totalRewardString := totalReward.String() + + // check if addr exists and update + var minerBalance string + addrExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(addrExistsStatement, minerAddr).Scan(&minerBalance) + + // Create addr or update existing balance + if err == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalRewardString, 0) + if insertErr != nil { + panic(insertErr) + } + } else if err != nil { + // Something went wrong panic + panic(err) + } else { + // Addr exists, update existing balance + var newBalance big.Int + newBalance.Add(addrInfo.balance, totalReward) + updateSQLStatement := `UPDATE accounts SET balance = ($1), txCountAccount = ($2) WHERE addr = ($3)` + _, updateErr := sqldb.Exec(updateSQLStatement, ) + } +} + +// @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: +// uncle blocks, account balance updates based on reorgs, diverges that get dropped. +// Reason for this is because the accounts are not deterministic like the block and tx hashes. +// @TODO: Calculate reward if there are uncles +// @TODO: Calculate mining reward (most likely retrieve higher up in the operations) +// @TODO: Calculate reorg +//func WriteMinerReward(db *leveldb.DB, block *types.Block) { +// var totalGas *big.Int +// var txs []string +// key := append([]byte("acc-")[:], block.Coinbase().Hash().Bytes()[:]...) +// for _, tx := range block.Transactions() { +// totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) +// } +//// retrievedData, err := db.Get(key, nil) +// if err != nil { +// // Assume time this account has had a tx +// // Balacne is exclusively minerreward + total gas from the block b/c no prior evm activity +// // Txs would be empty because they have not had any transactions on the EVM +// // @TODO: Calc mining reward +// //balance := totalGas.Add(totalGas, MINING_REWARD) +// balance := totalGas +// accData := ShyftAccountEntry{ +// Balance: balance, +// Txs: txs, +// } +// var encodedData bytes.Buffer +// encoder := gob.NewEncoder(&encodedData) +// if err := encoder.Encode(accData); err != nil { +// log.Crit("Faild to encode Miner Account data", "err", err) +// } +// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { +// log.Crit("Could not write the miner's first tx", "err", err) +// } +// } else { +// // The account has already have previous data stored due to activity in the EVM +// // Decode the data to update balance +// var decodedData ShyftAccountEntry +// d := gob.NewDecoder(bytes.NewBuffer(retrievedData)) +// if err := d.Decode(&decodedData); err != nil { +// log.Crit("Failed to decode miner data:", "err", err) +// } +// // Write new balance +// // @TODO: Calc mining reward +// // decodedData.Balance.Add(decodedData.Balance, totalGas.Add(totalGas, MINING_REWARD))) +// decodedData.Balance.Add(decodedData.Balance, totalGas) +// // Encode the data to be written back to the db +// var encodedData bytes.Buffer +// encoder := gob.NewEncoder(&encodedData) +// if err := encoder.Encode(decodedData); err != nil { +// log.Crit("Faild to encode Miner Account data", "err", err) +// } +// // Write newly encoded data back to the db +// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { +// log.Crit("Could not update miner account data", "err", err) +// } +// } +//} + + + +/////////// +// Getters +////////// +//GetAllBlocks returns []SBlock blocks for API +func GetAllBlocks(sqldb *sql.DB) string { + var arr blockRes + var blockArr string + rows, err := sqldb.Query(` + SELECT + hash, + coinbase, + gasused, + gaslimit, + txcount, + unclecount, + age, + number + FROM blocks`) + if err != nil { + fmt.Println("err") + } + defer rows.Close() + + for rows.Next() { + var hash string + var coinbase string + var gasUsed string + var gasLimit string + var txCount string + var uncleCount string + var age string + var num string + + err = rows.Scan( + &hash, + &coinbase, + &gasUsed, + &gasLimit, + &txCount, + &uncleCount, + &age, + &num, + ) + + arr.Blocks = append(arr.Blocks, SBlock{ + Hash: hash, + Coinbase: coinbase, + GasUsed: gasUsed, + GasLimit: gasLimit, + TxCount: txCount, + UncleCount: uncleCount, + Age: age, + Number: num, + }) + + blocks, _ := json.Marshal(arr.Blocks) + blocksFmt := string(blocks) + blockArr = blocksFmt + } + return blockArr +} + +//GetBlock queries to send single block info +//TODO provide blockHash arg passed from handler.go +func GetBlock(sqldb *sql.DB, blockNumber string) string { + sqlStatement := `SELECT * FROM blocks WHERE number=$1;` + row := sqldb.QueryRow(sqlStatement, blockNumber) + var hash string + var coinbase string + var gasUsed string + var gasLimit string + var txCount string + var uncleCount string + var age string + var parentHash string + var uncleHash string + var difficulty string + var size string + var nonce string + var num string + row.Scan( + &hash, + &coinbase, + &gasUsed, + &gasLimit, + &txCount, + &uncleCount, + &age, + &parentHash, + &uncleHash, + &difficulty, + &size, + &nonce, + &num,) + + block := SBlock{ + Hash: hash, + Coinbase: coinbase, + GasUsed: gasUsed, + GasLimit: gasLimit, + TxCount: txCount, + UncleCount: uncleCount, + Age: age, + ParentHash:parentHash, + UncleHash:uncleHash, + Difficulty:difficulty, + Size: size, + Nonce:nonce, + Number: num, + } + json, _ := json.Marshal(block) + return string(json) +} + +//GetAllTransactions getter fn for API +func GetAllTransactions(sqldb *sql.DB) string { + var arr txRes + var txx string + rows, err := sqldb.Query(` + SELECT + txhash, + to_addr, + from_addr, + blockhash, + blocknumber, + amount, + gasprice, + gas, + txfee, + nonce, + data + FROM txs`) + if err != nil { + fmt.Println("err") + } + defer rows.Close() + for rows.Next() { + var txhash string + var to_addr string + var from_addr string + var blockhash string + var blocknumber string + var amount uint64 + var gasprice uint64 + var gas uint64 + var txfee uint64 + var nonce uint64 + var data []byte + err = rows.Scan( + &txhash, + &to_addr, + &from_addr, + &blockhash, + &blocknumber, + &amount, + &gasprice, + &gas, + &txfee, + &nonce, + &data, + ) + + arr.TxEntry = append(arr.TxEntry, ShyftTxEntryPretty{ + TxHash: txhash, + To: to_addr, + From: from_addr, + BlockHash: blockhash, + BlockNumber: blocknumber, + Amount: amount, + GasPrice: gasprice, + Gas: gas, + Cost: txfee, + Nonce: nonce, + Data: data, + }) + + tx, _ := json.Marshal(arr.TxEntry) + newtx := string(tx) + txx = newtx + } + return txx +} + +//GetTransaction fn returns single tx +func GetTransaction(sqldb *sql.DB, txHash string) string { + sqlStatement := `SELECT * FROM txs WHERE txhash=$1;` + row := sqldb.QueryRow(sqlStatement, txHash) + var txhash string + var to_addr string + var from_addr string + var blockhash string + var blocknumber string + var amount uint64 + var gasprice uint64 + var gas uint64 + var txfee uint64 + var nonce uint64 + var data []byte + row.Scan( + &txhash, + &to_addr, + &from_addr, + &blockhash, + &amount, + &gasprice, + &gas, + &txfee, + &nonce, + &data) + tx := ShyftTxEntryPretty{ + TxHash: txhash, + To: to_addr, + From: from_addr, + BlockHash: blockhash, + BlockNumber: blocknumber, + Amount: amount, + GasPrice: gasprice, + Gas: gas, + Cost: txfee, + Nonce: nonce, + Data: data, + } + json, _ := json.Marshal(tx) + + return string(json) +} + +//GetAccount returns account balances +func GetAccount(sqldb *sql.DB, address string) string { + sqlStatement := `SELECT * FROM accounts WHERE addr=$1;` + row := sqldb.QueryRow(sqlStatement, address) + var addr string + var balance string + var txCountAccount string + row.Scan( + &addr, + &balance, + &txCountAccount) + + account := SAccounts{ + Addr: addr, + Balance: balance, + TxCountAccount: txCountAccount, + } + json, _ := json.Marshal(account) + return string(json) +} + +//GetAllAccounts returns all accounts and balances +func GetAllAccounts(sqldb *sql.DB) string { + var array accountRes + var accountsArr string + var txCountAccount string + accs, err := sqldb.Query(` + SELECT + addr, + balance, + txCountAccount + FROM accounts`) + if err != nil { + fmt.Println(err) + } + + defer accs.Close() + + for accs.Next() { + var addr string + var balance string + err = accs.Scan( + &addr, + &balance, + &txCountAccount, + ) + + array.AllAccounts = append(array.AllAccounts, SAccounts{ + Addr: addr, + Balance: balance, + TxCountAccount: txCountAccount, + }) + + accounts, _ := json.Marshal(array.AllAccounts) + accountsFmt := string(accounts) + accountsArr = accountsFmt + } + return accountsArr +} \ No newline at end of file diff --git a/shyftFullReset.sh b/shyftFullReset.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/shyftdb/database.go b/shyftdb/database.go deleted file mode 100644 index b30302a68a..0000000000 --- a/shyftdb/database.go +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package shyftdb - -import ( - "strconv" - "strings" - "sync" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" -) - -var OpenFileLimit = 64 - -type LDBDatabase struct { - fn string // filename for reporting - db *leveldb.DB // LevelDB instance - - compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter metrics.Meter // Meter for measuring the data read during compaction - compWriteMeter metrics.Meter // Meter for measuring the data written during compaction - diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read - diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written - - quitLock sync.Mutex // Mutex protecting the quit channel access - quitChan chan chan error // Quit channel to stop the metrics collection before closing the database - - log log.Logger // Contextual logger tracking the database path -} - -// NewLDBDatabase returns a LevelDB wrapped object. -func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { - logger := log.New("database", file) - - // Ensure we have some minimal caching and file guarantees - if cache < 16 { - cache = 16 - } - if handles < 16 { - handles = 16 - } - logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles) - - // Open the db and recover any potential corruptions - db, err := leveldb.OpenFile(file, &opt.Options{ - OpenFilesCacheCapacity: handles, - BlockCacheCapacity: cache / 2 * opt.MiB, - WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally - Filter: filter.NewBloomFilter(10), - }) - if _, corrupted := err.(*errors.ErrCorrupted); corrupted { - db, err = leveldb.RecoverFile(file, nil) - } - // (Re)check for errors and abort if opening of the db failed - if err != nil { - return nil, err - } - return &LDBDatabase{ - fn: file, - db: db, - log: logger, - }, nil -} - -// Path returns the path to the database directory. -func (db *LDBDatabase) Path() string { - return db.fn -} - -// Put puts the given key / value to the queue -func (db *LDBDatabase) Put(key []byte, value []byte) error { - // Generate the data to write to disk, update the meter and write - //value = rle.Compress(value) - - return db.db.Put(key, value, nil) -} - -func (db *LDBDatabase) Has(key []byte) (bool, error) { - return db.db.Has(key, nil) -} - -// Get returns the given key if it's present. -func (db *LDBDatabase) Get(key []byte) ([]byte, error) { - // Retrieve the key and increment the miss counter if not found - dat, err := db.db.Get(key, nil) - if err != nil { - return nil, err - } - return dat, nil - //return rle.Decompress(dat) -} - -// Delete deletes the key from the queue and database -func (db *LDBDatabase) Delete(key []byte) error { - // Execute the actual operation - return db.db.Delete(key, nil) -} - -func (db *LDBDatabase) NewIterator() iterator.Iterator { - return db.db.NewIterator(nil, nil) -} - -func (db *LDBDatabase) Close() { - // Stop the metrics collection to avoid internal database races - db.quitLock.Lock() - defer db.quitLock.Unlock() - - if db.quitChan != nil { - errc := make(chan error) - db.quitChan <- errc - if err := <-errc; err != nil { - db.log.Error("Metrics collection failed", "err", err) - } - } - err := db.db.Close() - if err == nil { - db.log.Info("Database closed") - } else { - db.log.Error("Failed to close database", "err", err) - } -} - -func (db *LDBDatabase) LDB() *leveldb.DB { - return db.db -} - -// Meter configures the database metrics collectors and -func (db *LDBDatabase) Meter(prefix string) { - // Short circuit metering if the metrics system is disabled - if !metrics.Enabled { - return - } - // Initialize all the metrics collector at the requested prefix - db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) - db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) - db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) - db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) - db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) - - // Create a quit channel for the periodic collector and run it - db.quitLock.Lock() - db.quitChan = make(chan chan error) - db.quitLock.Unlock() - - go db.meter(3 * time.Second) -} - -// meter periodically retrieves internal leveldb counters and reports them to -// the metrics subsystem. -// -// This is how a stats table look like (currently): -// Compactions -// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) -// -------+------------+---------------+---------------+---------------+--------------- -// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 -// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 -// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 -// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 -// -// This is how the iostats look like (currently): -// Read(MB):3895.04860 Write(MB):3654.64712 -func (db *LDBDatabase) meter(refresh time.Duration) { - // Create the counters to store current and previous compaction values - compactions := make([][]float64, 2) - for i := 0; i < 2; i++ { - compactions[i] = make([]float64, 3) - } - // Create storage for iostats. - var iostats [2]float64 - // Iterate ad infinitum and collect the stats - for i := 1; ; i++ { - // Retrieve the database stats - stats, err := db.db.GetProperty("leveldb.stats") - if err != nil { - db.log.Error("Failed to read database stats", "err", err) - return - } - // Find the compaction table, skip the header - lines := strings.Split(stats, "\n") - for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { - lines = lines[1:] - } - if len(lines) <= 3 { - db.log.Error("Compaction table not found") - return - } - lines = lines[3:] - - // Iterate over all the table rows, and accumulate the entries - for j := 0; j < len(compactions[i%2]); j++ { - compactions[i%2][j] = 0 - } - for _, line := range lines { - parts := strings.Split(line, "|") - if len(parts) != 6 { - break - } - for idx, counter := range parts[3:] { - value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) - if err != nil { - db.log.Error("Compaction entry parsing failed", "err", err) - return - } - compactions[i%2][idx] += value - } - } - // Update all the requested meters - if db.compTimeMeter != nil { - db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) - } - if db.compReadMeter != nil { - db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) - } - if db.compWriteMeter != nil { - db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) - } - - // Retrieve the database iostats. - ioStats, err := db.db.GetProperty("leveldb.iostats") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - return - } - parts := strings.Split(ioStats, " ") - if len(parts) < 2 { - db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - return - } - r := strings.Split(parts[0], ":") - if len(r) < 2 { - db.log.Error("Bad syntax of read entry", "entry", parts[0]) - return - } - read, err := strconv.ParseFloat(r[1], 64) - if err != nil { - db.log.Error("Read entry parsing failed", "err", err) - return - } - w := strings.Split(parts[1], ":") - if len(w) < 2 { - db.log.Error("Bad syntax of write entry", "entry", parts[1]) - return - } - write, err := strconv.ParseFloat(w[1], 64) - if err != nil { - db.log.Error("Write entry parsing failed", "err", err) - return - } - if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((read - iostats[0]) * 1024 * 1024)) - } - if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((write - iostats[1]) * 1024 * 1024)) - } - iostats[0] = read - iostats[1] = write - - // Sleep a bit, then repeat the stats collection - select { - case errc := <-db.quitChan: - // Quit requesting, stop hammering the database - errc <- nil - return - - case <-time.After(refresh): - // Timeout, gather a new set of stats - } - } -} - -func (db *LDBDatabase) NewBatch() Batch { - return &ldbBatch{db: db.db, b: new(leveldb.Batch)} -} - -type ldbBatch struct { - db *leveldb.DB - b *leveldb.Batch - size int -} - -func (b *ldbBatch) Put(key, value []byte) error { - b.b.Put(key, value) - b.size += len(value) - return nil -} - -func (b *ldbBatch) Write() error { - return b.db.Write(b.b, nil) -} - -func (b *ldbBatch) ValueSize() int { - return b.size -} - -func (b *ldbBatch) Reset() { - b.b.Reset() - b.size = 0 -} - -type table struct { - db Database - prefix string -} - -// NewTable returns a Database object that prefixes all keys with a given -// string. -func NewTable(db Database, prefix string) Database { - return &table{ - db: db, - prefix: prefix, - } -} - -func (dt *table) Put(key []byte, value []byte) error { - return dt.db.Put(append([]byte(dt.prefix), key...), value) -} - -func (dt *table) Has(key []byte) (bool, error) { - return dt.db.Has(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Get(key []byte) ([]byte, error) { - return dt.db.Get(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Delete(key []byte) error { - return dt.db.Delete(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Close() { - // Do nothing; don't close the underlying DB. -} - -type tableBatch struct { - batch Batch - prefix string -} - -// NewTableBatch returns a Batch object which prefixes all keys with a given string. -func NewTableBatch(db Database, prefix string) Batch { - return &tableBatch{db.NewBatch(), prefix} -} - -func (dt *table) NewBatch() Batch { - return &tableBatch{dt.db.NewBatch(), dt.prefix} -} - -func (tb *tableBatch) Put(key, value []byte) error { - return tb.batch.Put(append([]byte(tb.prefix), key...), value) -} - -func (tb *tableBatch) Write() error { - return tb.batch.Write() -} - -func (tb *tableBatch) ValueSize() int { - return tb.batch.ValueSize() -} - -func (tb *tableBatch) Reset() { - tb.batch.Reset() -} diff --git a/shyftdb/interface.go b/shyftdb/interface.go deleted file mode 100644 index fe361cc791..0000000000 --- a/shyftdb/interface.go +++ /dev/null @@ -1,31 +0,0 @@ - -package shyftdb - -// Code using batches should try to add this much data to the batch. -// The value was determined empirically. -const IdealBatchSize = 100 * 1024 - -// Putter wraps the database write operation supported by both batches and regular databases. -type Putter interface { - Put(key []byte, value []byte) error -} - -// Database wraps all database operations. All methods are safe for concurrent use. -type Database interface { - Putter - Get(key []byte) ([]byte, error) - Has(key []byte) (bool, error) - Delete(key []byte) error - Close() - NewBatch() Batch -} - -// Batch is a write-only database that commits changes to its host database -// when Write is called. Batch cannot be used concurrently. -type Batch interface { - Putter - ValueSize() int // amount of data in the batch - Write() error - // Reset resets the batch for reuse - Reset() -} diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index af3475d4d6..c821f50423 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -12,6 +12,7 @@ import ( "log" _ "github.com/lib/pq" + "reflect" ) //SBlock type @@ -98,13 +99,13 @@ type SendAndReceive struct { } //WriteBlock writes to block info to sql db - func WriteBlock(sqldb *sql.DB, block *types.Block, receipts []*types.Receipt) error { +func WriteBlock(sqldb *sql.DB, block *types.Block, receipts []*types.Receipt) error { //Need to create field in postgres db isContract : True || False //Need to fix nonce numeric issue (attempt to reproduce and record) //Need to update tx To Field where null with Contract Address //Need to update account table with that Contract Address //Need to update AccountNonce and Balance of Tx To field || Contract Address - + WriteMinerRewards(sqldb,block) coinbase := block.Header().Coinbase.String() number := block.Header().Number.String() gasUsed := block.Header().GasUsed @@ -418,19 +419,44 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s return sendAndReceiveData, balanceReceiver, balanceSender, accountNonceReceiver, accountNonceSender } -//func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { -// var totalGas big.Int -// //var txs []string -// -// fmt.Println("this is BLOCK.UNCLE", block.Uncles()) -// fmt.Println("this is BLOCK UNCLE HASH", block.UncleHash().String()) -// fmt.Println("this is BLOCK.TRANSACTIONS", block.Transactions()) -// fmt.Println("this is BLOCK TOTAL GAS", block.GasUsed()) -// -// for _, tx := range block.Transactions() { -// totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) -// } -//} +func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { + minerAddr := block.Coinbase().String() + fmt.Println("\n\n\t\t", minerAddr, "\n\n") + // Calculate the total gas used in the block + var totalGas big.Int + for _, tx := range block.Transactions() { + totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) + } + + //TODO: Calculate Block Reward + // totalReward := totalGas.Add(&totalGas, MINER_REWARD) + totalReward := totalGas + totalRewardString := totalReward.String() + + // check if addr exists and update + var minerBalance string + addrExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(addrExistsStatement, minerAddr).Scan(&minerBalance) + + // Create addr or update existing balance + if err == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalRewardString, 0) + if insertErr != nil { + panic(insertErr) + } + } else if err != nil { + // Something went wrong panic + panic(err) + } else { + // Addr exists, update existing balance + var newBalance big.Int + newBalance.Add(addrInfo.balance, totalReward) + updateSQLStatement := `UPDATE accounts SET balance = ($1), txCountAccount = ($2) WHERE addr = ($3)` + _, updateErr := sqldb.Exec(updateSQLStatement, ) + } +} // @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: // uncle blocks, account balance updates based on reorgs, diverges that get dropped. diff --git a/simulations/sendTransactions.js b/simulations/sendTransactions.js index b819a8338a..95ce1b71c5 100644 --- a/simulations/sendTransactions.js +++ b/simulations/sendTransactions.js @@ -1,29 +1,25 @@ -var firstAccount = web3.eth.accounts[0] -var secondAccount = web3.eth.accounts[1] -var thirdAccount = web3.eth.accounts[2] - for (var i = 0; i < 1; i++) { console.log('\t\t' + (i + 1) + ' - Transactions') web3.eth.sendTransaction({ - from: web3.eth.accounts[1], + from: web3.eth.accounts[3], to: web3.eth.accounts[2], value: 5, gas: 50000, gasPrice: 20 }); - // web3.eth.sendTransaction({ - // from: web3.eth.accounts[0], - // to: web3.eth.accounts[2], - // value: 291, - // gas: 50000, - // gasPrice: 20 - // }); - // - // web3.eth.sendTransaction({ - // from: web3.eth.accounts[0], - // to: web3.eth.accounts[1], - // value: 53039, - // gas: 50000, - // gasPrice: 20 - // }); + web3.eth.sendTransaction({ + from: web3.eth.accounts[3], + to: web3.eth.accounts[2], + value: 291, + gas: 50000, + gasPrice: 20 + }); + + web3.eth.sendTransaction({ + from: web3.eth.accounts[3], + to: web3.eth.accounts[1], + value: 53039, + gas: 50000, + gasPrice: 20 + }); } \ No newline at end of file From 83c8f4c2c6557cf8b6e119066dad377df7897a55 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 16 May 2018 15:50:30 -0400 Subject: [PATCH 2/9] Add script to drop tables, re-init tables & reset shyftGeth --- shyftFullReset.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shyftFullReset.sh b/shyftFullReset.sh index e69de29bb2..6fbcbc1683 100644 --- a/shyftFullReset.sh +++ b/shyftFullReset.sh @@ -0,0 +1,5 @@ +cd ./shyftDb/postgres_setup +sh drop_tables.sh && sh init_tables.sh +cd .. +cd .. +sh resetShyftGeth.sh && sh initShyftGeth.sh \ No newline at end of file From bc24aef25e69d7e311f5af0fc077a34d7ad9c37b Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 16 May 2018 16:30:54 -0400 Subject: [PATCH 3/9] Base miner reward updating --- shyftDb/shyft_database_util.go | 32 ++++++++++++++++++++++---------- shyftdb/shyft_database_util.go | 32 ++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index c821f50423..afb5abc898 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -12,7 +12,6 @@ import ( "log" _ "github.com/lib/pq" - "reflect" ) //SBlock type @@ -421,17 +420,16 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { minerAddr := block.Coinbase().String() - fmt.Println("\n\n\t\t", minerAddr, "\n\n") + // Calculate the total gas used in the block - var totalGas big.Int + totalGas := new(big.Int) for _, tx := range block.Transactions() { - totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) + totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) } //TODO: Calculate Block Reward // totalReward := totalGas.Add(&totalGas, MINER_REWARD) totalReward := totalGas - totalRewardString := totalReward.String() // check if addr exists and update var minerBalance string @@ -442,7 +440,9 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { if err == sql.ErrNoRows { // Addr does not exist, thus create new entry createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` - _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalRewardString, 0) + + // We convert totalReward into a string and postgres converts into number + _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalReward.String(), 0) if insertErr != nil { panic(insertErr) } @@ -451,10 +451,22 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { panic(err) } else { // Addr exists, update existing balance - var newBalance big.Int - newBalance.Add(addrInfo.balance, totalReward) - updateSQLStatement := `UPDATE accounts SET balance = ($1), txCountAccount = ($2) WHERE addr = ($3)` - _, updateErr := sqldb.Exec(updateSQLStatement, ) + bigMinerBalance := new(big.Int) + bigMinerBalance, err := bigMinerBalance.SetString(minerBalance, 0) + if !err { + panic(err) + } + newBalance := new(big.Int) + newBalance.Add(newBalance, bigMinerBalance) + newBalance.Add(newBalance, totalReward) + + updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` + + // We convert totalReward into a string and postgres converts into number + _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), minerAddr) + if updateErr != nil { + panic(updateErr) + } } } diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index c821f50423..afb5abc898 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -12,7 +12,6 @@ import ( "log" _ "github.com/lib/pq" - "reflect" ) //SBlock type @@ -421,17 +420,16 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { minerAddr := block.Coinbase().String() - fmt.Println("\n\n\t\t", minerAddr, "\n\n") + // Calculate the total gas used in the block - var totalGas big.Int + totalGas := new(big.Int) for _, tx := range block.Transactions() { - totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) + totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) } //TODO: Calculate Block Reward // totalReward := totalGas.Add(&totalGas, MINER_REWARD) totalReward := totalGas - totalRewardString := totalReward.String() // check if addr exists and update var minerBalance string @@ -442,7 +440,9 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { if err == sql.ErrNoRows { // Addr does not exist, thus create new entry createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` - _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalRewardString, 0) + + // We convert totalReward into a string and postgres converts into number + _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalReward.String(), 0) if insertErr != nil { panic(insertErr) } @@ -451,10 +451,22 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { panic(err) } else { // Addr exists, update existing balance - var newBalance big.Int - newBalance.Add(addrInfo.balance, totalReward) - updateSQLStatement := `UPDATE accounts SET balance = ($1), txCountAccount = ($2) WHERE addr = ($3)` - _, updateErr := sqldb.Exec(updateSQLStatement, ) + bigMinerBalance := new(big.Int) + bigMinerBalance, err := bigMinerBalance.SetString(minerBalance, 0) + if !err { + panic(err) + } + newBalance := new(big.Int) + newBalance.Add(newBalance, bigMinerBalance) + newBalance.Add(newBalance, totalReward) + + updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` + + // We convert totalReward into a string and postgres converts into number + _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), minerAddr) + if updateErr != nil { + panic(updateErr) + } } } From ecf7349a2bee119ec244331529fcdb8da9aa5be8 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 17 May 2018 15:16:46 -0400 Subject: [PATCH 4/9] Add shyft conduit reward --- shyftDb/shyft_database_util.go | 64 ++++++++++++++++++++++++++-------- shyftdb/shyft_database_util.go | 64 ++++++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 28 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index afb5abc898..820c9a0b89 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -12,6 +12,7 @@ import ( "log" _ "github.com/lib/pq" + Rewards "github.com/ethereum/go-ethereum/consensus/ethash" ) //SBlock type @@ -418,25 +419,65 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s return sendAndReceiveData, balanceReceiver, balanceSender, accountNonceReceiver, accountNonceSender } +// @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: +// uncle blocks, account balance updates based on reorgs, diverges that get dropped. +// Reason for this is because the accounts are not deterministic like the block and tx hashes. +// @TODO: Calculate reward if there are uncles +// @TODO: Calculate reorg func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { minerAddr := block.Coinbase().String() - + shyftConduitAddress := Rewards.ShyftNetworkConduitAddress.String() // Calculate the total gas used in the block totalGas := new(big.Int) for _, tx := range block.Transactions() { totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) } - //TODO: Calculate Block Reward - // totalReward := totalGas.Add(&totalGas, MINER_REWARD) - totalReward := totalGas + totalReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) - // check if addr exists and update + // Check if shyft conduit exists + var shyftBalance string + shyftExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + shyftErr := sqldb.QueryRow(shyftExistsStatement, shyftConduitAddress).Scan(­ftBalance) + + // Create shyft conduit addr or update exisitng balance + if shyftErr == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + + // We convert ShyftConduitReward into a string and postgres converts into number + _, insertErr := sqldb.Exec(createAddrSqlStatement, shyftConduitAddress, Rewards.ShyftNetworkBlockReward.String(), 0) + if insertErr != nil { + panic(insertErr) + } + } else if shyftErr != nil { + panic(shyftErr) + } else { + // shyftAddr exists, update existing balance + bigShyftBalance := new(big.Int) + bigShyftBalance, err := bigShyftBalance.SetString(shyftBalance, 0) + if !err { + panic(err) + } + newBalance := new(big.Int) + newBalance.Add(newBalance, bigShyftBalance) + newBalance.Add(newBalance, totalReward) + + updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` + + // We convert totalReward into a string and postgres converts into number + _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), shyftConduitAddress) + if updateErr != nil { + panic(updateErr) + } + } + + // Check if miner exists var minerBalance string - addrExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` - err := sqldb.QueryRow(addrExistsStatement, minerAddr).Scan(&minerBalance) + minerExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(minerExistsStatement, minerAddr).Scan(&minerBalance) - // Create addr or update existing balance + // Create miner or update existing balance if err == sql.ErrNoRows { // Addr does not exist, thus create new entry createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` @@ -470,12 +511,7 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { } } -// @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: -// uncle blocks, account balance updates based on reorgs, diverges that get dropped. -// Reason for this is because the accounts are not deterministic like the block and tx hashes. -// @TODO: Calculate reward if there are uncles -// @TODO: Calculate mining reward (most likely retrieve higher up in the operations) -// @TODO: Calculate reorg + //func WriteMinerReward(db *leveldb.DB, block *types.Block) { // var totalGas *big.Int // var txs []string diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index afb5abc898..820c9a0b89 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -12,6 +12,7 @@ import ( "log" _ "github.com/lib/pq" + Rewards "github.com/ethereum/go-ethereum/consensus/ethash" ) //SBlock type @@ -418,25 +419,65 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s return sendAndReceiveData, balanceReceiver, balanceSender, accountNonceReceiver, accountNonceSender } +// @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: +// uncle blocks, account balance updates based on reorgs, diverges that get dropped. +// Reason for this is because the accounts are not deterministic like the block and tx hashes. +// @TODO: Calculate reward if there are uncles +// @TODO: Calculate reorg func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { minerAddr := block.Coinbase().String() - + shyftConduitAddress := Rewards.ShyftNetworkConduitAddress.String() // Calculate the total gas used in the block totalGas := new(big.Int) for _, tx := range block.Transactions() { totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) } - //TODO: Calculate Block Reward - // totalReward := totalGas.Add(&totalGas, MINER_REWARD) - totalReward := totalGas + totalReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) - // check if addr exists and update + // Check if shyft conduit exists + var shyftBalance string + shyftExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + shyftErr := sqldb.QueryRow(shyftExistsStatement, shyftConduitAddress).Scan(­ftBalance) + + // Create shyft conduit addr or update exisitng balance + if shyftErr == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + + // We convert ShyftConduitReward into a string and postgres converts into number + _, insertErr := sqldb.Exec(createAddrSqlStatement, shyftConduitAddress, Rewards.ShyftNetworkBlockReward.String(), 0) + if insertErr != nil { + panic(insertErr) + } + } else if shyftErr != nil { + panic(shyftErr) + } else { + // shyftAddr exists, update existing balance + bigShyftBalance := new(big.Int) + bigShyftBalance, err := bigShyftBalance.SetString(shyftBalance, 0) + if !err { + panic(err) + } + newBalance := new(big.Int) + newBalance.Add(newBalance, bigShyftBalance) + newBalance.Add(newBalance, totalReward) + + updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` + + // We convert totalReward into a string and postgres converts into number + _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), shyftConduitAddress) + if updateErr != nil { + panic(updateErr) + } + } + + // Check if miner exists var minerBalance string - addrExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` - err := sqldb.QueryRow(addrExistsStatement, minerAddr).Scan(&minerBalance) + minerExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(minerExistsStatement, minerAddr).Scan(&minerBalance) - // Create addr or update existing balance + // Create miner or update existing balance if err == sql.ErrNoRows { // Addr does not exist, thus create new entry createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` @@ -470,12 +511,7 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { } } -// @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: -// uncle blocks, account balance updates based on reorgs, diverges that get dropped. -// Reason for this is because the accounts are not deterministic like the block and tx hashes. -// @TODO: Calculate reward if there are uncles -// @TODO: Calculate mining reward (most likely retrieve higher up in the operations) -// @TODO: Calculate reorg + //func WriteMinerReward(db *leveldb.DB, block *types.Block) { // var totalGas *big.Int // var txs []string From 41748a394dd4522dcb5a9e4ce1838fd4f04416e8 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 17 May 2018 15:18:24 -0400 Subject: [PATCH 5/9] Removed old code add snippet for how uncle reward should be calculated --- shyftDb/shyft_database_util.go | 68 +++++++--------------------------- shyftdb/shyft_database_util.go | 68 +++++++--------------------------- 2 files changed, 28 insertions(+), 108 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index 820c9a0b89..8142000a12 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -509,63 +509,23 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { panic(updateErr) } } + // Accumulate the rewards for the miner and any included uncles + //reward := new(big.Int).Set(blockReward) + //r := new(big.Int) + //for _, uncle := range uncles { + // r.Add(uncle.Number, big8) + // r.Sub(r, header.Number) + // r.Mul(r, blockReward) + // r.Div(r, big8) + // state.AddBalance(uncle.Coinbase, r) + // + // r.Div(blockReward, big32) + // reward.Add(reward, r) + //} + //state.AddBalance(header.Coinbase, reward) } -//func WriteMinerReward(db *leveldb.DB, block *types.Block) { -// var totalGas *big.Int -// var txs []string -// key := append([]byte("acc-")[:], block.Coinbase().Hash().Bytes()[:]...) -// for _, tx := range block.Transactions() { -// totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) -// } -//// retrievedData, err := db.Get(key, nil) -// if err != nil { -// // Assume time this account has had a tx -// // Balacne is exclusively minerreward + total gas from the block b/c no prior evm activity -// // Txs would be empty because they have not had any transactions on the EVM -// // @TODO: Calc mining reward -// //balance := totalGas.Add(totalGas, MINING_REWARD) -// balance := totalGas -// accData := ShyftAccountEntry{ -// Balance: balance, -// Txs: txs, -// } -// var encodedData bytes.Buffer -// encoder := gob.NewEncoder(&encodedData) -// if err := encoder.Encode(accData); err != nil { -// log.Crit("Faild to encode Miner Account data", "err", err) -// } -// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { -// log.Crit("Could not write the miner's first tx", "err", err) -// } -// } else { -// // The account has already have previous data stored due to activity in the EVM -// // Decode the data to update balance -// var decodedData ShyftAccountEntry -// d := gob.NewDecoder(bytes.NewBuffer(retrievedData)) -// if err := d.Decode(&decodedData); err != nil { -// log.Crit("Failed to decode miner data:", "err", err) -// } -// // Write new balance -// // @TODO: Calc mining reward -// // decodedData.Balance.Add(decodedData.Balance, totalGas.Add(totalGas, MINING_REWARD))) -// decodedData.Balance.Add(decodedData.Balance, totalGas) -// // Encode the data to be written back to the db -// var encodedData bytes.Buffer -// encoder := gob.NewEncoder(&encodedData) -// if err := encoder.Encode(decodedData); err != nil { -// log.Crit("Faild to encode Miner Account data", "err", err) -// } -// // Write newly encoded data back to the db -// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { -// log.Crit("Could not update miner account data", "err", err) -// } -// } -//} - - - /////////// // Getters ////////// diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index 820c9a0b89..8142000a12 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -509,63 +509,23 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { panic(updateErr) } } + // Accumulate the rewards for the miner and any included uncles + //reward := new(big.Int).Set(blockReward) + //r := new(big.Int) + //for _, uncle := range uncles { + // r.Add(uncle.Number, big8) + // r.Sub(r, header.Number) + // r.Mul(r, blockReward) + // r.Div(r, big8) + // state.AddBalance(uncle.Coinbase, r) + // + // r.Div(blockReward, big32) + // reward.Add(reward, r) + //} + //state.AddBalance(header.Coinbase, reward) } -//func WriteMinerReward(db *leveldb.DB, block *types.Block) { -// var totalGas *big.Int -// var txs []string -// key := append([]byte("acc-")[:], block.Coinbase().Hash().Bytes()[:]...) -// for _, tx := range block.Transactions() { -// totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) -// } -//// retrievedData, err := db.Get(key, nil) -// if err != nil { -// // Assume time this account has had a tx -// // Balacne is exclusively minerreward + total gas from the block b/c no prior evm activity -// // Txs would be empty because they have not had any transactions on the EVM -// // @TODO: Calc mining reward -// //balance := totalGas.Add(totalGas, MINING_REWARD) -// balance := totalGas -// accData := ShyftAccountEntry{ -// Balance: balance, -// Txs: txs, -// } -// var encodedData bytes.Buffer -// encoder := gob.NewEncoder(&encodedData) -// if err := encoder.Encode(accData); err != nil { -// log.Crit("Faild to encode Miner Account data", "err", err) -// } -// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { -// log.Crit("Could not write the miner's first tx", "err", err) -// } -// } else { -// // The account has already have previous data stored due to activity in the EVM -// // Decode the data to update balance -// var decodedData ShyftAccountEntry -// d := gob.NewDecoder(bytes.NewBuffer(retrievedData)) -// if err := d.Decode(&decodedData); err != nil { -// log.Crit("Failed to decode miner data:", "err", err) -// } -// // Write new balance -// // @TODO: Calc mining reward -// // decodedData.Balance.Add(decodedData.Balance, totalGas.Add(totalGas, MINING_REWARD))) -// decodedData.Balance.Add(decodedData.Balance, totalGas) -// // Encode the data to be written back to the db -// var encodedData bytes.Buffer -// encoder := gob.NewEncoder(&encodedData) -// if err := encoder.Encode(decodedData); err != nil { -// log.Crit("Faild to encode Miner Account data", "err", err) -// } -// // Write newly encoded data back to the db -// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { -// log.Crit("Could not update miner account data", "err", err) -// } -// } -//} - - - /////////// // Getters ////////// From 5dee19495c1732c1b2fe7bbb1f664bb217bbd48f Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 17 May 2018 16:35:20 -0400 Subject: [PATCH 6/9] Refactor reward system into two seperate functions --- shyftDb/shyft_database_util.go | 116 ++++++++++++--------------------- shyftdb/shyft_database_util.go | 116 ++++++++++++--------------------- 2 files changed, 86 insertions(+), 146 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index 8142000a12..2feaa055ca 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -433,82 +433,12 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) } - totalReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) + totalMinerReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) - // Check if shyft conduit exists - var shyftBalance string - shyftExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` - shyftErr := sqldb.QueryRow(shyftExistsStatement, shyftConduitAddress).Scan(­ftBalance) + StoreReward(sqldb, minerAddr, totalMinerReward) + StoreReward(sqldb, shyftConduitAddress, Rewards.ShyftNetworkBlockReward) - // Create shyft conduit addr or update exisitng balance - if shyftErr == sql.ErrNoRows { - // Addr does not exist, thus create new entry - createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` - // We convert ShyftConduitReward into a string and postgres converts into number - _, insertErr := sqldb.Exec(createAddrSqlStatement, shyftConduitAddress, Rewards.ShyftNetworkBlockReward.String(), 0) - if insertErr != nil { - panic(insertErr) - } - } else if shyftErr != nil { - panic(shyftErr) - } else { - // shyftAddr exists, update existing balance - bigShyftBalance := new(big.Int) - bigShyftBalance, err := bigShyftBalance.SetString(shyftBalance, 0) - if !err { - panic(err) - } - newBalance := new(big.Int) - newBalance.Add(newBalance, bigShyftBalance) - newBalance.Add(newBalance, totalReward) - - updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` - - // We convert totalReward into a string and postgres converts into number - _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), shyftConduitAddress) - if updateErr != nil { - panic(updateErr) - } - } - - // Check if miner exists - var minerBalance string - minerExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` - err := sqldb.QueryRow(minerExistsStatement, minerAddr).Scan(&minerBalance) - - // Create miner or update existing balance - if err == sql.ErrNoRows { - // Addr does not exist, thus create new entry - createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` - - // We convert totalReward into a string and postgres converts into number - _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalReward.String(), 0) - if insertErr != nil { - panic(insertErr) - } - } else if err != nil { - // Something went wrong panic - panic(err) - } else { - // Addr exists, update existing balance - bigMinerBalance := new(big.Int) - bigMinerBalance, err := bigMinerBalance.SetString(minerBalance, 0) - if !err { - panic(err) - } - newBalance := new(big.Int) - newBalance.Add(newBalance, bigMinerBalance) - newBalance.Add(newBalance, totalReward) - - updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` - - // We convert totalReward into a string and postgres converts into number - _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), minerAddr) - if updateErr != nil { - panic(updateErr) - } - } // Accumulate the rewards for the miner and any included uncles //reward := new(big.Int).Set(blockReward) //r := new(big.Int) @@ -525,6 +455,46 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { //state.AddBalance(header.Coinbase, reward) } +func StoreReward(sqldb *sql.DB, address string, reward *big.Int) { + // Check if address exists + var addressBalance string + addressExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(addressExistsStatement, address).Scan(&addressBalance) + + if err == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddressSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + + // We convert totalReward into a string and postgres converts into number + _, insertErr := sqldb.Exec(createAddressSqlStatement, address, reward.String(), 0) + if insertErr != nil { + panic(insertErr) + } + return + } else if err != nil { + // Something went wrong panic + panic(err) + } else { + // Addr exists, update existing balance + bigBalance := new(big.Int) + bigBalance, err := bigBalance.SetString(addressBalance, 0) + if !err { + panic(err) + } + newBalance := new(big.Int) + newBalance.Add(newBalance, bigBalance) + newBalance.Add(newBalance, reward) + + updateAddressSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` + + // We convert totalReward into a string and postgres converts into number + _, updateErr := sqldb.Exec(updateAddressSQLStatement, newBalance.String(), address) + if updateErr != nil { + panic(updateErr) + } + return + } +} /////////// // Getters diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index 8142000a12..2feaa055ca 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -433,82 +433,12 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) } - totalReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) + totalMinerReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) - // Check if shyft conduit exists - var shyftBalance string - shyftExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` - shyftErr := sqldb.QueryRow(shyftExistsStatement, shyftConduitAddress).Scan(­ftBalance) + StoreReward(sqldb, minerAddr, totalMinerReward) + StoreReward(sqldb, shyftConduitAddress, Rewards.ShyftNetworkBlockReward) - // Create shyft conduit addr or update exisitng balance - if shyftErr == sql.ErrNoRows { - // Addr does not exist, thus create new entry - createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` - // We convert ShyftConduitReward into a string and postgres converts into number - _, insertErr := sqldb.Exec(createAddrSqlStatement, shyftConduitAddress, Rewards.ShyftNetworkBlockReward.String(), 0) - if insertErr != nil { - panic(insertErr) - } - } else if shyftErr != nil { - panic(shyftErr) - } else { - // shyftAddr exists, update existing balance - bigShyftBalance := new(big.Int) - bigShyftBalance, err := bigShyftBalance.SetString(shyftBalance, 0) - if !err { - panic(err) - } - newBalance := new(big.Int) - newBalance.Add(newBalance, bigShyftBalance) - newBalance.Add(newBalance, totalReward) - - updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` - - // We convert totalReward into a string and postgres converts into number - _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), shyftConduitAddress) - if updateErr != nil { - panic(updateErr) - } - } - - // Check if miner exists - var minerBalance string - minerExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` - err := sqldb.QueryRow(minerExistsStatement, minerAddr).Scan(&minerBalance) - - // Create miner or update existing balance - if err == sql.ErrNoRows { - // Addr does not exist, thus create new entry - createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` - - // We convert totalReward into a string and postgres converts into number - _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalReward.String(), 0) - if insertErr != nil { - panic(insertErr) - } - } else if err != nil { - // Something went wrong panic - panic(err) - } else { - // Addr exists, update existing balance - bigMinerBalance := new(big.Int) - bigMinerBalance, err := bigMinerBalance.SetString(minerBalance, 0) - if !err { - panic(err) - } - newBalance := new(big.Int) - newBalance.Add(newBalance, bigMinerBalance) - newBalance.Add(newBalance, totalReward) - - updateSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` - - // We convert totalReward into a string and postgres converts into number - _, updateErr := sqldb.Exec(updateSQLStatement, newBalance.String(), minerAddr) - if updateErr != nil { - panic(updateErr) - } - } // Accumulate the rewards for the miner and any included uncles //reward := new(big.Int).Set(blockReward) //r := new(big.Int) @@ -525,6 +455,46 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { //state.AddBalance(header.Coinbase, reward) } +func StoreReward(sqldb *sql.DB, address string, reward *big.Int) { + // Check if address exists + var addressBalance string + addressExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(addressExistsStatement, address).Scan(&addressBalance) + + if err == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddressSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + + // We convert totalReward into a string and postgres converts into number + _, insertErr := sqldb.Exec(createAddressSqlStatement, address, reward.String(), 0) + if insertErr != nil { + panic(insertErr) + } + return + } else if err != nil { + // Something went wrong panic + panic(err) + } else { + // Addr exists, update existing balance + bigBalance := new(big.Int) + bigBalance, err := bigBalance.SetString(addressBalance, 0) + if !err { + panic(err) + } + newBalance := new(big.Int) + newBalance.Add(newBalance, bigBalance) + newBalance.Add(newBalance, reward) + + updateAddressSQLStatement := `UPDATE accounts SET balance = ($1) WHERE addr = ($2)` + + // We convert totalReward into a string and postgres converts into number + _, updateErr := sqldb.Exec(updateAddressSQLStatement, newBalance.String(), address) + if updateErr != nil { + panic(updateErr) + } + return + } +} /////////// // Getters From 64453a4d5b1cac2c27665054174bbedd9e7cb414 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 17 May 2018 16:53:48 -0400 Subject: [PATCH 7/9] uncle rewards in process --- shyftDb/shyft_database_util.go | 2 +- shyftdb/shyft_database_util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index 2feaa055ca..93dbaca165 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -438,7 +438,7 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { StoreReward(sqldb, minerAddr, totalMinerReward) StoreReward(sqldb, shyftConduitAddress, Rewards.ShyftNetworkBlockReward) - + //https://ethereum.stackexchange.com/questions/27172/different-uncles-reward // Accumulate the rewards for the miner and any included uncles //reward := new(big.Int).Set(blockReward) //r := new(big.Int) diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index 2feaa055ca..93dbaca165 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -438,7 +438,7 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { StoreReward(sqldb, minerAddr, totalMinerReward) StoreReward(sqldb, shyftConduitAddress, Rewards.ShyftNetworkBlockReward) - + //https://ethereum.stackexchange.com/questions/27172/different-uncles-reward // Accumulate the rewards for the miner and any included uncles //reward := new(big.Int).Set(blockReward) //r := new(big.Int) From d19ff3191b0a9cef327f8e15e7f028b4ad00fc68 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 17 May 2018 17:25:44 -0400 Subject: [PATCH 8/9] Uncle reward logic implimented --- shyftDb/shyft_database_util.go | 38 ++++++++++++++++++++-------------- shyftdb/shyft_database_util.go | 38 ++++++++++++++++++++-------------- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index 93dbaca165..5b8353fb1b 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -435,24 +435,30 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { totalMinerReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) + // References: + // https://ethereum.stackexchange.com/questions/27172/different-uncles-reward + // line 551 in consensus.go (shyft_go-ethereum/consensus/ethash/consensus.go) + // Some weird constants to avoid constant memory allocs for them. + var big8 = big.NewInt(8) + var uncleRewards []*big.Int + var uncleAddrs []string + + // uncleReward is overwritten after each iteration + uncleReward := new(big.Int) + for _, uncle := range block.Uncles() { + uncleReward.Add(uncle.Number, big8) + uncleReward.Sub(uncleReward, block.Number()) + uncleReward.Mul(uncleReward, Rewards.ShyftMinerBlockReward) + uncleReward.Div(uncleReward, big8) + uncleRewards = append(uncleRewards, uncleReward) + uncleAddrs = append(uncleAddrs, uncle.Coinbase.String()) + } + StoreReward(sqldb, minerAddr, totalMinerReward) StoreReward(sqldb, shyftConduitAddress, Rewards.ShyftNetworkBlockReward) - - //https://ethereum.stackexchange.com/questions/27172/different-uncles-reward - // Accumulate the rewards for the miner and any included uncles - //reward := new(big.Int).Set(blockReward) - //r := new(big.Int) - //for _, uncle := range uncles { - // r.Add(uncle.Number, big8) - // r.Sub(r, header.Number) - // r.Mul(r, blockReward) - // r.Div(r, big8) - // state.AddBalance(uncle.Coinbase, r) - // - // r.Div(blockReward, big32) - // reward.Add(reward, r) - //} - //state.AddBalance(header.Coinbase, reward) + for i := 0; i < len(uncleAddrs); i++ { + StoreReward(sqldb, uncleAddrs[i], uncleRewards[i]) + } } func StoreReward(sqldb *sql.DB, address string, reward *big.Int) { diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index 93dbaca165..5b8353fb1b 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -435,24 +435,30 @@ func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { totalMinerReward := totalGas.Add(totalGas, Rewards.ShyftMinerBlockReward) + // References: + // https://ethereum.stackexchange.com/questions/27172/different-uncles-reward + // line 551 in consensus.go (shyft_go-ethereum/consensus/ethash/consensus.go) + // Some weird constants to avoid constant memory allocs for them. + var big8 = big.NewInt(8) + var uncleRewards []*big.Int + var uncleAddrs []string + + // uncleReward is overwritten after each iteration + uncleReward := new(big.Int) + for _, uncle := range block.Uncles() { + uncleReward.Add(uncle.Number, big8) + uncleReward.Sub(uncleReward, block.Number()) + uncleReward.Mul(uncleReward, Rewards.ShyftMinerBlockReward) + uncleReward.Div(uncleReward, big8) + uncleRewards = append(uncleRewards, uncleReward) + uncleAddrs = append(uncleAddrs, uncle.Coinbase.String()) + } + StoreReward(sqldb, minerAddr, totalMinerReward) StoreReward(sqldb, shyftConduitAddress, Rewards.ShyftNetworkBlockReward) - - //https://ethereum.stackexchange.com/questions/27172/different-uncles-reward - // Accumulate the rewards for the miner and any included uncles - //reward := new(big.Int).Set(blockReward) - //r := new(big.Int) - //for _, uncle := range uncles { - // r.Add(uncle.Number, big8) - // r.Sub(r, header.Number) - // r.Mul(r, blockReward) - // r.Div(r, big8) - // state.AddBalance(uncle.Coinbase, r) - // - // r.Div(blockReward, big32) - // reward.Add(reward, r) - //} - //state.AddBalance(header.Coinbase, reward) + for i := 0; i < len(uncleAddrs); i++ { + StoreReward(sqldb, uncleAddrs[i], uncleRewards[i]) + } } func StoreReward(sqldb *sql.DB, address string, reward *big.Int) { From 30860fa7e0f781095169332c55643543b48705a9 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 17 May 2018 17:32:38 -0400 Subject: [PATCH 9/9] remove todo --- shyftDb/shyft_database_util.go | 1 - shyftdb/shyft_database_util.go | 1 - 2 files changed, 2 deletions(-) diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go index 5b8353fb1b..136e248d48 100644 --- a/shyftDb/shyft_database_util.go +++ b/shyftDb/shyft_database_util.go @@ -422,7 +422,6 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s // @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: // uncle blocks, account balance updates based on reorgs, diverges that get dropped. // Reason for this is because the accounts are not deterministic like the block and tx hashes. -// @TODO: Calculate reward if there are uncles // @TODO: Calculate reorg func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { minerAddr := block.Coinbase().String() diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index 5b8353fb1b..136e248d48 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -422,7 +422,6 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s // @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: // uncle blocks, account balance updates based on reorgs, diverges that get dropped. // Reason for this is because the accounts are not deterministic like the block and tx hashes. -// @TODO: Calculate reward if there are uncles // @TODO: Calculate reorg func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { minerAddr := block.Coinbase().String()