From 56dd351ac6352eef8fb0352ea17aa6d2c98f9aa8 Mon Sep 17 00:00:00 2001 From: priom Date: Wed, 16 May 2018 15:48:16 -0400 Subject: [PATCH] initial miner reward --- shyftDb/postgres_setup/create_tables.psql | 5 + shyftDb/postgres_setup/drop_tables.psql | 3 +- shyftDb/shyft_database_util.go | 806 ++++++++++++++++++++++ shyftFullReset.sh | 0 shyftdb/database.go | 383 ---------- shyftdb/interface.go | 31 - shyftdb/shyft_database_util.go | 56 +- simulations/sendTransactions.js | 36 +- 8 files changed, 870 insertions(+), 450 deletions(-) create mode 100644 shyftDb/shyft_database_util.go create mode 100644 shyftFullReset.sh delete mode 100644 shyftdb/database.go delete mode 100644 shyftdb/interface.go diff --git a/shyftDb/postgres_setup/create_tables.psql b/shyftDb/postgres_setup/create_tables.psql index 80afba9790..c24638c450 100644 --- a/shyftDb/postgres_setup/create_tables.psql +++ b/shyftDb/postgres_setup/create_tables.psql @@ -37,4 +37,9 @@ CREATE TABLE IF NOT EXISTS accounts ( CREATE TABLE IF NOT EXISTS contracts ( txHash text +); + +CREATE TABLE IF NOT EXISTS mined_blocks ( + blockNumber bigint, + addr text ); \ No newline at end of file diff --git a/shyftDb/postgres_setup/drop_tables.psql b/shyftDb/postgres_setup/drop_tables.psql index 5df13ed64c..2921358098 100644 --- a/shyftDb/postgres_setup/drop_tables.psql +++ b/shyftDb/postgres_setup/drop_tables.psql @@ -1,4 +1,5 @@ DROP TABLE txs; DROP TABLE blocks; DROP TABLE accounts; -DROP TABLE contracts; \ No newline at end of file +DROP TABLE contracts; +DROP TABLE mined_blocks; \ No newline at end of file diff --git a/shyftDb/shyft_database_util.go b/shyftDb/shyft_database_util.go new file mode 100644 index 0000000000..c821f50423 --- /dev/null +++ b/shyftDb/shyft_database_util.go @@ -0,0 +1,806 @@ +package shyftdb + +import ( + "encoding/json" + "fmt" + "math/big" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "time" + "strconv" + "database/sql" + "log" + + _ "github.com/lib/pq" + "reflect" +) + +//SBlock type +type SBlock struct { + Hash string + Coinbase string + Number string + GasUsed string + GasLimit string + TxCount string + UncleCount string + Age string + ParentHash string + UncleHash string + Difficulty string + Size string + Nonce string +} + +//blockRes struct +type blockRes struct { + hash string + coinbase string + number string + Blocks []SBlock +} + +type SAccounts struct { + Addr string + Balance string + TxCountAccount string +} + +type accountRes struct { + addr string + balance string + AllAccounts []SAccounts +} + +//ShyftTxEntry structure +type ShyftTxEntry struct { + TxHash common.Hash + To *common.Address + From *common.Address + BlockHash string + Amount *big.Int + GasPrice *big.Int + Gas uint64 + Cost *big.Int + Nonce uint64 + Data []byte +} + +type txRes struct { + TxEntry []ShyftTxEntryPretty +} + +type ShyftTxEntryPretty struct { + TxHash string + To string + From string + BlockHash string + BlockNumber string + Amount uint64 + GasPrice uint64 + Gas uint64 + Cost uint64 + Nonce uint64 + Data []byte +} + +type ShyftAccountEntry struct { + Balance string + Txs []string +} + +type SendAndReceive struct { + To string + From string + Amount string + Address string + Balance string + TxCountAccount string +} + +//WriteBlock writes to block info to sql db +func WriteBlock(sqldb *sql.DB, block *types.Block, receipts []*types.Receipt) error { + //Need to create field in postgres db isContract : True || False + //Need to fix nonce numeric issue (attempt to reproduce and record) + //Need to update tx To Field where null with Contract Address + //Need to update account table with that Contract Address + //Need to update AccountNonce and Balance of Tx To field || Contract Address + WriteMinerRewards(sqldb,block) + coinbase := block.Header().Coinbase.String() + number := block.Header().Number.String() + gasUsed := block.Header().GasUsed + gasLimit := block.Header().GasLimit + txCount := block.Transactions().Len() + uncleCount := len(block.Uncles()) + parentHash := block.ParentHash().String() + uncleHash := block.UncleHash().String() + blockDifficulty := block.Difficulty().String() + blockSize := block.Size().String() + blockNonce := block.Nonce() + + // Convert the receipts into their storage form and serialize them + storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) + for i, receipt := range receipts { + storageReceipts[i] = (*types.ReceiptForStorage)(receipt) + var txHashFromReciept = (*types.ReceiptForStorage)(receipt).TxHash + var statusFromReciept = (*types.ReceiptForStorage)(receipt).Status + var contractAddressFromReciept = (*types.ReceiptForStorage)(receipt).ContractAddress + if statusFromReciept == 1 { + fmt.Println("THIS IS statusFromReciept", "SUCCESS", statusFromReciept) + } + if statusFromReciept == 0 { + fmt.Println("THIS IS statusFromReciept", "FAIL", statusFromReciept) + } + + if block.Transactions()[0].To() == nil { + updateSQLStatement := `UPDATE txs SET to_addr = ($2) WHERE txHash = ($1)` + _, error := sqldb.Exec(updateSQLStatement, txHashFromReciept.String(), contractAddressFromReciept.String()) + if error != nil { + panic(error) + } + } + + + fmt.Println("THIS IS txHashFromReciept", txHashFromReciept.String()) + fmt.Println("THIS IS contractAddressFromReciept", contractAddressFromReciept.String()) + } + + i, err := strconv.ParseInt(block.Time().String(), 10, 64) + if err != nil { + panic(err) + } + age := time.Unix(i, 0) + + sqlStatement := `INSERT INTO blocks(hash, coinbase, number, gasUsed, gasLimit, txCount, uncleCount, age, parentHash, uncleHash, difficulty, size, nonce) VALUES(($1), ($2), ($3), ($4), ($5), ($6), ($7), ($8), ($9), ($10), ($11), ($12),($13)) RETURNING number` + qerr := sqldb.QueryRow(sqlStatement, block.Header().Hash().Hex(), coinbase, number, gasUsed, gasLimit, txCount, uncleCount, age, parentHash, uncleHash, blockDifficulty, blockSize, blockNonce).Scan(&number) + if qerr != nil { + panic(qerr) + } + + if block.Transactions().Len() > 0 { + for _, tx := range block.Transactions() { + //WriteMinerRewards(sqldb, block) + WriteTransactions(sqldb, tx, block.Header().Hash(), block.Header().Number.String()) + if block.Transactions()[0].To() != nil { + WriteFromBalance(sqldb, tx) + } + if block.Transactions()[0].To() == nil { + WriteContractBalance(sqldb, tx) + WriteContractsTxHashReferences(sqldb, tx) + } + } + } + return nil +} + +//WriteTransactions writes to sqldb +func WriteTransactions(sqldb *sql.DB, tx *types.Transaction, blockHash common.Hash, blockNumber string) error { + txData := ShyftTxEntry{ + TxHash: tx.Hash(), + From: tx.From(), + To: tx.To(), + BlockHash: blockHash.Hex(), + Amount: tx.Value(), + Cost: tx.Cost(), + GasPrice: tx.GasPrice(), + Gas: tx.Gas(), + Nonce: tx.Nonce(), + Data: tx.Data(), + } + + txHash := txData.TxHash.Hex() + from := txData.From.Hex() + blockHasher := txData.BlockHash + amount := txData.Amount.String() + gasPrice := txData.GasPrice.String() + txFee := txData.Cost.String() + nonce := txData.Nonce + gas := txData.Gas + data := txData.Data + to := txData.To + var isContract bool + if (to == nil){ + var retNonce string + isContract = true + sqlStatement := `INSERT INTO txs(txhash, from_addr, blockhash, blockNumber, amount, gasprice, gas, txfee, nonce, isContract, data) VALUES(($1), ($2), ($3), ($4), ($5), ($6), ($7), ($8), ($9), ($10), ($11)) RETURNING nonce` + qerr := sqldb.QueryRow(sqlStatement, txHash, from, blockHasher, blockNumber, amount, gasPrice, gas, txFee, nonce, isContract, data).Scan(&retNonce) + + if qerr != nil { + panic(qerr) + } + } else { + var retNonce string + isContract = false + sqlStatement := `INSERT INTO txs(txhash, from_addr, to_addr, blockhash, blockNumber, amount, gasprice, gas, txfee, nonce, isContract, data) VALUES(($1), ($2), ($3), ($4), ($5), ($6), ($7), ($8), ($9), ($10), ($11), ($12)) RETURNING nonce` + qerr := sqldb.QueryRow(sqlStatement, txHash, from, to.Hex(), blockHasher, blockNumber, amount, gasPrice, gas, txFee, nonce, isContract, data).Scan(&retNonce) + + if qerr != nil { + panic(qerr) + } + } + + return nil +} + +func WriteContractsTxHashReferences(sqldb *sql.DB, tx *types.Transaction) error { + txHash := tx.Hash().Hex() + + sqlStatement := `INSERT INTO contracts(txHash) VALUES(($1)) RETURNING txHash` + insertErr := sqldb.QueryRow(sqlStatement, txHash).Scan(&txHash) + if insertErr != nil { + panic(insertErr) + } + return nil +} + +func WriteContractBalance(sqldb *sql.DB, tx *types.Transaction) error { + sendAndReceiveData,balanceSen,accountNonceSen := WriteContractBalanceHelper(sqldb, tx) + fromAddr := sendAndReceiveData.From + amount := sendAndReceiveData.Amount + balanceSender := balanceSen + + var response string + sqlExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(sqlExistsStatement, fromAddr).Scan(&response) + switch { + case err == sql.ErrNoRows: + fmt.Println("NO ROWS RAN") + //i, err := strconv.Atoi(accountNonceSen) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println("accountnonce", i) + //fmt.Println(reflect.TypeOf(i)) + sqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + insertErr := sqldb.QueryRow(sqlStatement, fromAddr, amount, accountNonceSen).Scan(&fromAddr) + if insertErr != nil { + panic(insertErr) + } + case err != nil: + log.Fatal(err) + default: + var newBalanceSender big.Int + var newAccountNonceSender big.Int + var nonceIncrement = big.NewInt(1) + updateSQLStatement := `UPDATE accounts SET balance = ($2), txCountAccount = ($3) WHERE addr = ($1)` + + s := new(big.Int) + _, error := fmt.Sscan(balanceSender, s) + if error != nil { + log.Println("error scanning value:", error) + } + + accountS := new(big.Int) + _, errors := fmt.Sscan(accountNonceSen, accountS) + if errors != nil { + log.Println("error scanning value:", error) + } + + newBalanceSender.Sub(s, tx.Value()) + newAccountNonceSender.Add(accountS, nonceIncrement) + + _, err = sqldb.Exec(updateSQLStatement, fromAddr, newBalanceSender.String(), newAccountNonceSender.String()) + if err != nil { + panic(err) + } + } + return nil +} + +func WriteContractBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, string, string) { + sendAndReceiveData := SendAndReceive{ + From: tx.From().Hex(), + Amount: tx.Value().String(), + } + + fromAddr := sendAndReceiveData.From + getAccountBalanceSender:= GetAccount(sqldb, fromAddr) + + var senderBalance SendAndReceive + if err := json.Unmarshal([]byte(getAccountBalanceSender), &senderBalance); err != nil { + log.Fatal(err) + } + balanceSender := senderBalance.Balance + accountNonceSender := senderBalance.TxCountAccount + + return sendAndReceiveData, balanceSender, accountNonceSender +} + +//WriteFromBalance writes senders balance to accounts db +func WriteFromBalance(sqldb *sql.DB, tx *types.Transaction) error { + //IF to address is nil (which means its contract creation) + //Need to create a condition (flag) check and then change how the nonce increment works + sendAndReceiveData, balanceRec, balanceSen, accountNonceRec, accountNonceSen := WriteBalanceHelper(sqldb, tx) + toAddr := sendAndReceiveData.To + fromAddr := sendAndReceiveData.From + amount := sendAndReceiveData.Amount + balanceReceiver := balanceRec + balanceSender := balanceSen + + var response string + sqlExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(sqlExistsStatement, toAddr).Scan(&response) + switch { + case err == sql.ErrNoRows: + fmt.Println("NO ROWS RAN") + i, err := strconv.Atoi(accountNonceRec) + if err != nil { + fmt.Println(err) + } + //fmt.Println("accountnonce", i) + //fmt.Println(reflect.TypeOf(i)) + sqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + insertErr := sqldb.QueryRow(sqlStatement, toAddr, amount, i).Scan(&toAddr) + if insertErr != nil { + panic(insertErr) + } + case err != nil: + log.Fatal(err) + default: + var newBalanceReceiver big.Int + var newBalanceSender big.Int + var newAccountNonceReceiver big.Int + var newAccountNonceSender big.Int + var nonceIncrement = big.NewInt(1) + updateSQLStatement := `UPDATE accounts SET balance = ($2), txCountAccount = ($3) WHERE addr = ($1)` + + r := new(big.Int) + _, err := fmt.Sscan(balanceReceiver, r) + if err != nil { + log.Println("error scanning value:", err) + } + + s := new(big.Int) + _, error := fmt.Sscan(balanceSender, s) + if error != nil { + log.Println("error scanning value:", error) + } + + accountR := new(big.Int) + _, er := fmt.Sscan(accountNonceRec, accountR) + if er != nil { + log.Println("error scanning value:", er) + } + + accountS := new(big.Int) + _, errors := fmt.Sscan(accountNonceSen, accountS) + if errors != nil { + log.Println("error scanning value:", error) + } + + newBalanceReceiver.Add(r, tx.Value()) + newBalanceSender.Sub(s, tx.Value()) + + newAccountNonceReceiver.Add(accountR, nonceIncrement) + newAccountNonceSender.Add(accountS, nonceIncrement) + + _, err = sqldb.Exec(updateSQLStatement, toAddr, newBalanceReceiver.String(), newAccountNonceReceiver.String()) + if err != nil { + panic(err) + } + + _, err = sqldb.Exec(updateSQLStatement, fromAddr, newBalanceSender.String(), newAccountNonceSender.String()) + if err != nil { + panic(err) + } + } + return nil +} + +func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, string, string, string, string) { + sendAndReceiveData := SendAndReceive{ + To: tx.To().Hex(), + From: tx.From().Hex(), + Amount: tx.Value().String(), + } + + toAddr := sendAndReceiveData.To + fromAddr := sendAndReceiveData.From + + getAccountBalanceReceiver := GetAccount(sqldb, toAddr) + getAccountBalanceSender:= GetAccount(sqldb, fromAddr) + + var receiverBalance SendAndReceive + if err := json.Unmarshal([]byte(getAccountBalanceReceiver), &receiverBalance); err != nil { + log.Fatal(err) + } + + var senderBalance SendAndReceive + if err := json.Unmarshal([]byte(getAccountBalanceSender), &senderBalance); err != nil { + log.Fatal(err) + } + + balanceReceiver := receiverBalance.Balance + balanceSender := senderBalance.Balance + + accountNonceReceiver := receiverBalance.TxCountAccount + accountNonceSender := senderBalance.TxCountAccount + + return sendAndReceiveData, balanceReceiver, balanceSender, accountNonceReceiver, accountNonceSender +} + +func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { + minerAddr := block.Coinbase().String() + fmt.Println("\n\n\t\t", minerAddr, "\n\n") + // Calculate the total gas used in the block + var totalGas big.Int + for _, tx := range block.Transactions() { + totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) + } + + //TODO: Calculate Block Reward + // totalReward := totalGas.Add(&totalGas, MINER_REWARD) + totalReward := totalGas + totalRewardString := totalReward.String() + + // check if addr exists and update + var minerBalance string + addrExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(addrExistsStatement, minerAddr).Scan(&minerBalance) + + // Create addr or update existing balance + if err == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalRewardString, 0) + if insertErr != nil { + panic(insertErr) + } + } else if err != nil { + // Something went wrong panic + panic(err) + } else { + // Addr exists, update existing balance + var newBalance big.Int + newBalance.Add(addrInfo.balance, totalReward) + updateSQLStatement := `UPDATE accounts SET balance = ($1), txCountAccount = ($2) WHERE addr = ($3)` + _, updateErr := sqldb.Exec(updateSQLStatement, ) + } +} + +// @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: +// uncle blocks, account balance updates based on reorgs, diverges that get dropped. +// Reason for this is because the accounts are not deterministic like the block and tx hashes. +// @TODO: Calculate reward if there are uncles +// @TODO: Calculate mining reward (most likely retrieve higher up in the operations) +// @TODO: Calculate reorg +//func WriteMinerReward(db *leveldb.DB, block *types.Block) { +// var totalGas *big.Int +// var txs []string +// key := append([]byte("acc-")[:], block.Coinbase().Hash().Bytes()[:]...) +// for _, tx := range block.Transactions() { +// totalGas.Add(totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) +// } +//// retrievedData, err := db.Get(key, nil) +// if err != nil { +// // Assume time this account has had a tx +// // Balacne is exclusively minerreward + total gas from the block b/c no prior evm activity +// // Txs would be empty because they have not had any transactions on the EVM +// // @TODO: Calc mining reward +// //balance := totalGas.Add(totalGas, MINING_REWARD) +// balance := totalGas +// accData := ShyftAccountEntry{ +// Balance: balance, +// Txs: txs, +// } +// var encodedData bytes.Buffer +// encoder := gob.NewEncoder(&encodedData) +// if err := encoder.Encode(accData); err != nil { +// log.Crit("Faild to encode Miner Account data", "err", err) +// } +// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { +// log.Crit("Could not write the miner's first tx", "err", err) +// } +// } else { +// // The account has already have previous data stored due to activity in the EVM +// // Decode the data to update balance +// var decodedData ShyftAccountEntry +// d := gob.NewDecoder(bytes.NewBuffer(retrievedData)) +// if err := d.Decode(&decodedData); err != nil { +// log.Crit("Failed to decode miner data:", "err", err) +// } +// // Write new balance +// // @TODO: Calc mining reward +// // decodedData.Balance.Add(decodedData.Balance, totalGas.Add(totalGas, MINING_REWARD))) +// decodedData.Balance.Add(decodedData.Balance, totalGas) +// // Encode the data to be written back to the db +// var encodedData bytes.Buffer +// encoder := gob.NewEncoder(&encodedData) +// if err := encoder.Encode(decodedData); err != nil { +// log.Crit("Faild to encode Miner Account data", "err", err) +// } +// // Write newly encoded data back to the db +// if err := db.Put(key, encodedData.Bytes(), nil); err != nil { +// log.Crit("Could not update miner account data", "err", err) +// } +// } +//} + + + +/////////// +// Getters +////////// +//GetAllBlocks returns []SBlock blocks for API +func GetAllBlocks(sqldb *sql.DB) string { + var arr blockRes + var blockArr string + rows, err := sqldb.Query(` + SELECT + hash, + coinbase, + gasused, + gaslimit, + txcount, + unclecount, + age, + number + FROM blocks`) + if err != nil { + fmt.Println("err") + } + defer rows.Close() + + for rows.Next() { + var hash string + var coinbase string + var gasUsed string + var gasLimit string + var txCount string + var uncleCount string + var age string + var num string + + err = rows.Scan( + &hash, + &coinbase, + &gasUsed, + &gasLimit, + &txCount, + &uncleCount, + &age, + &num, + ) + + arr.Blocks = append(arr.Blocks, SBlock{ + Hash: hash, + Coinbase: coinbase, + GasUsed: gasUsed, + GasLimit: gasLimit, + TxCount: txCount, + UncleCount: uncleCount, + Age: age, + Number: num, + }) + + blocks, _ := json.Marshal(arr.Blocks) + blocksFmt := string(blocks) + blockArr = blocksFmt + } + return blockArr +} + +//GetBlock queries to send single block info +//TODO provide blockHash arg passed from handler.go +func GetBlock(sqldb *sql.DB, blockNumber string) string { + sqlStatement := `SELECT * FROM blocks WHERE number=$1;` + row := sqldb.QueryRow(sqlStatement, blockNumber) + var hash string + var coinbase string + var gasUsed string + var gasLimit string + var txCount string + var uncleCount string + var age string + var parentHash string + var uncleHash string + var difficulty string + var size string + var nonce string + var num string + row.Scan( + &hash, + &coinbase, + &gasUsed, + &gasLimit, + &txCount, + &uncleCount, + &age, + &parentHash, + &uncleHash, + &difficulty, + &size, + &nonce, + &num,) + + block := SBlock{ + Hash: hash, + Coinbase: coinbase, + GasUsed: gasUsed, + GasLimit: gasLimit, + TxCount: txCount, + UncleCount: uncleCount, + Age: age, + ParentHash:parentHash, + UncleHash:uncleHash, + Difficulty:difficulty, + Size: size, + Nonce:nonce, + Number: num, + } + json, _ := json.Marshal(block) + return string(json) +} + +//GetAllTransactions getter fn for API +func GetAllTransactions(sqldb *sql.DB) string { + var arr txRes + var txx string + rows, err := sqldb.Query(` + SELECT + txhash, + to_addr, + from_addr, + blockhash, + blocknumber, + amount, + gasprice, + gas, + txfee, + nonce, + data + FROM txs`) + if err != nil { + fmt.Println("err") + } + defer rows.Close() + for rows.Next() { + var txhash string + var to_addr string + var from_addr string + var blockhash string + var blocknumber string + var amount uint64 + var gasprice uint64 + var gas uint64 + var txfee uint64 + var nonce uint64 + var data []byte + err = rows.Scan( + &txhash, + &to_addr, + &from_addr, + &blockhash, + &blocknumber, + &amount, + &gasprice, + &gas, + &txfee, + &nonce, + &data, + ) + + arr.TxEntry = append(arr.TxEntry, ShyftTxEntryPretty{ + TxHash: txhash, + To: to_addr, + From: from_addr, + BlockHash: blockhash, + BlockNumber: blocknumber, + Amount: amount, + GasPrice: gasprice, + Gas: gas, + Cost: txfee, + Nonce: nonce, + Data: data, + }) + + tx, _ := json.Marshal(arr.TxEntry) + newtx := string(tx) + txx = newtx + } + return txx +} + +//GetTransaction fn returns single tx +func GetTransaction(sqldb *sql.DB, txHash string) string { + sqlStatement := `SELECT * FROM txs WHERE txhash=$1;` + row := sqldb.QueryRow(sqlStatement, txHash) + var txhash string + var to_addr string + var from_addr string + var blockhash string + var blocknumber string + var amount uint64 + var gasprice uint64 + var gas uint64 + var txfee uint64 + var nonce uint64 + var data []byte + row.Scan( + &txhash, + &to_addr, + &from_addr, + &blockhash, + &amount, + &gasprice, + &gas, + &txfee, + &nonce, + &data) + tx := ShyftTxEntryPretty{ + TxHash: txhash, + To: to_addr, + From: from_addr, + BlockHash: blockhash, + BlockNumber: blocknumber, + Amount: amount, + GasPrice: gasprice, + Gas: gas, + Cost: txfee, + Nonce: nonce, + Data: data, + } + json, _ := json.Marshal(tx) + + return string(json) +} + +//GetAccount returns account balances +func GetAccount(sqldb *sql.DB, address string) string { + sqlStatement := `SELECT * FROM accounts WHERE addr=$1;` + row := sqldb.QueryRow(sqlStatement, address) + var addr string + var balance string + var txCountAccount string + row.Scan( + &addr, + &balance, + &txCountAccount) + + account := SAccounts{ + Addr: addr, + Balance: balance, + TxCountAccount: txCountAccount, + } + json, _ := json.Marshal(account) + return string(json) +} + +//GetAllAccounts returns all accounts and balances +func GetAllAccounts(sqldb *sql.DB) string { + var array accountRes + var accountsArr string + var txCountAccount string + accs, err := sqldb.Query(` + SELECT + addr, + balance, + txCountAccount + FROM accounts`) + if err != nil { + fmt.Println(err) + } + + defer accs.Close() + + for accs.Next() { + var addr string + var balance string + err = accs.Scan( + &addr, + &balance, + &txCountAccount, + ) + + array.AllAccounts = append(array.AllAccounts, SAccounts{ + Addr: addr, + Balance: balance, + TxCountAccount: txCountAccount, + }) + + accounts, _ := json.Marshal(array.AllAccounts) + accountsFmt := string(accounts) + accountsArr = accountsFmt + } + return accountsArr +} \ No newline at end of file diff --git a/shyftFullReset.sh b/shyftFullReset.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/shyftdb/database.go b/shyftdb/database.go deleted file mode 100644 index b30302a68a..0000000000 --- a/shyftdb/database.go +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package shyftdb - -import ( - "strconv" - "strings" - "sync" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" -) - -var OpenFileLimit = 64 - -type LDBDatabase struct { - fn string // filename for reporting - db *leveldb.DB // LevelDB instance - - compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter metrics.Meter // Meter for measuring the data read during compaction - compWriteMeter metrics.Meter // Meter for measuring the data written during compaction - diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read - diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written - - quitLock sync.Mutex // Mutex protecting the quit channel access - quitChan chan chan error // Quit channel to stop the metrics collection before closing the database - - log log.Logger // Contextual logger tracking the database path -} - -// NewLDBDatabase returns a LevelDB wrapped object. -func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { - logger := log.New("database", file) - - // Ensure we have some minimal caching and file guarantees - if cache < 16 { - cache = 16 - } - if handles < 16 { - handles = 16 - } - logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles) - - // Open the db and recover any potential corruptions - db, err := leveldb.OpenFile(file, &opt.Options{ - OpenFilesCacheCapacity: handles, - BlockCacheCapacity: cache / 2 * opt.MiB, - WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally - Filter: filter.NewBloomFilter(10), - }) - if _, corrupted := err.(*errors.ErrCorrupted); corrupted { - db, err = leveldb.RecoverFile(file, nil) - } - // (Re)check for errors and abort if opening of the db failed - if err != nil { - return nil, err - } - return &LDBDatabase{ - fn: file, - db: db, - log: logger, - }, nil -} - -// Path returns the path to the database directory. -func (db *LDBDatabase) Path() string { - return db.fn -} - -// Put puts the given key / value to the queue -func (db *LDBDatabase) Put(key []byte, value []byte) error { - // Generate the data to write to disk, update the meter and write - //value = rle.Compress(value) - - return db.db.Put(key, value, nil) -} - -func (db *LDBDatabase) Has(key []byte) (bool, error) { - return db.db.Has(key, nil) -} - -// Get returns the given key if it's present. -func (db *LDBDatabase) Get(key []byte) ([]byte, error) { - // Retrieve the key and increment the miss counter if not found - dat, err := db.db.Get(key, nil) - if err != nil { - return nil, err - } - return dat, nil - //return rle.Decompress(dat) -} - -// Delete deletes the key from the queue and database -func (db *LDBDatabase) Delete(key []byte) error { - // Execute the actual operation - return db.db.Delete(key, nil) -} - -func (db *LDBDatabase) NewIterator() iterator.Iterator { - return db.db.NewIterator(nil, nil) -} - -func (db *LDBDatabase) Close() { - // Stop the metrics collection to avoid internal database races - db.quitLock.Lock() - defer db.quitLock.Unlock() - - if db.quitChan != nil { - errc := make(chan error) - db.quitChan <- errc - if err := <-errc; err != nil { - db.log.Error("Metrics collection failed", "err", err) - } - } - err := db.db.Close() - if err == nil { - db.log.Info("Database closed") - } else { - db.log.Error("Failed to close database", "err", err) - } -} - -func (db *LDBDatabase) LDB() *leveldb.DB { - return db.db -} - -// Meter configures the database metrics collectors and -func (db *LDBDatabase) Meter(prefix string) { - // Short circuit metering if the metrics system is disabled - if !metrics.Enabled { - return - } - // Initialize all the metrics collector at the requested prefix - db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) - db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) - db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) - db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) - db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) - - // Create a quit channel for the periodic collector and run it - db.quitLock.Lock() - db.quitChan = make(chan chan error) - db.quitLock.Unlock() - - go db.meter(3 * time.Second) -} - -// meter periodically retrieves internal leveldb counters and reports them to -// the metrics subsystem. -// -// This is how a stats table look like (currently): -// Compactions -// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) -// -------+------------+---------------+---------------+---------------+--------------- -// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 -// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 -// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 -// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 -// -// This is how the iostats look like (currently): -// Read(MB):3895.04860 Write(MB):3654.64712 -func (db *LDBDatabase) meter(refresh time.Duration) { - // Create the counters to store current and previous compaction values - compactions := make([][]float64, 2) - for i := 0; i < 2; i++ { - compactions[i] = make([]float64, 3) - } - // Create storage for iostats. - var iostats [2]float64 - // Iterate ad infinitum and collect the stats - for i := 1; ; i++ { - // Retrieve the database stats - stats, err := db.db.GetProperty("leveldb.stats") - if err != nil { - db.log.Error("Failed to read database stats", "err", err) - return - } - // Find the compaction table, skip the header - lines := strings.Split(stats, "\n") - for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { - lines = lines[1:] - } - if len(lines) <= 3 { - db.log.Error("Compaction table not found") - return - } - lines = lines[3:] - - // Iterate over all the table rows, and accumulate the entries - for j := 0; j < len(compactions[i%2]); j++ { - compactions[i%2][j] = 0 - } - for _, line := range lines { - parts := strings.Split(line, "|") - if len(parts) != 6 { - break - } - for idx, counter := range parts[3:] { - value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) - if err != nil { - db.log.Error("Compaction entry parsing failed", "err", err) - return - } - compactions[i%2][idx] += value - } - } - // Update all the requested meters - if db.compTimeMeter != nil { - db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) - } - if db.compReadMeter != nil { - db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) - } - if db.compWriteMeter != nil { - db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) - } - - // Retrieve the database iostats. - ioStats, err := db.db.GetProperty("leveldb.iostats") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - return - } - parts := strings.Split(ioStats, " ") - if len(parts) < 2 { - db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - return - } - r := strings.Split(parts[0], ":") - if len(r) < 2 { - db.log.Error("Bad syntax of read entry", "entry", parts[0]) - return - } - read, err := strconv.ParseFloat(r[1], 64) - if err != nil { - db.log.Error("Read entry parsing failed", "err", err) - return - } - w := strings.Split(parts[1], ":") - if len(w) < 2 { - db.log.Error("Bad syntax of write entry", "entry", parts[1]) - return - } - write, err := strconv.ParseFloat(w[1], 64) - if err != nil { - db.log.Error("Write entry parsing failed", "err", err) - return - } - if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((read - iostats[0]) * 1024 * 1024)) - } - if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((write - iostats[1]) * 1024 * 1024)) - } - iostats[0] = read - iostats[1] = write - - // Sleep a bit, then repeat the stats collection - select { - case errc := <-db.quitChan: - // Quit requesting, stop hammering the database - errc <- nil - return - - case <-time.After(refresh): - // Timeout, gather a new set of stats - } - } -} - -func (db *LDBDatabase) NewBatch() Batch { - return &ldbBatch{db: db.db, b: new(leveldb.Batch)} -} - -type ldbBatch struct { - db *leveldb.DB - b *leveldb.Batch - size int -} - -func (b *ldbBatch) Put(key, value []byte) error { - b.b.Put(key, value) - b.size += len(value) - return nil -} - -func (b *ldbBatch) Write() error { - return b.db.Write(b.b, nil) -} - -func (b *ldbBatch) ValueSize() int { - return b.size -} - -func (b *ldbBatch) Reset() { - b.b.Reset() - b.size = 0 -} - -type table struct { - db Database - prefix string -} - -// NewTable returns a Database object that prefixes all keys with a given -// string. -func NewTable(db Database, prefix string) Database { - return &table{ - db: db, - prefix: prefix, - } -} - -func (dt *table) Put(key []byte, value []byte) error { - return dt.db.Put(append([]byte(dt.prefix), key...), value) -} - -func (dt *table) Has(key []byte) (bool, error) { - return dt.db.Has(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Get(key []byte) ([]byte, error) { - return dt.db.Get(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Delete(key []byte) error { - return dt.db.Delete(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Close() { - // Do nothing; don't close the underlying DB. -} - -type tableBatch struct { - batch Batch - prefix string -} - -// NewTableBatch returns a Batch object which prefixes all keys with a given string. -func NewTableBatch(db Database, prefix string) Batch { - return &tableBatch{db.NewBatch(), prefix} -} - -func (dt *table) NewBatch() Batch { - return &tableBatch{dt.db.NewBatch(), dt.prefix} -} - -func (tb *tableBatch) Put(key, value []byte) error { - return tb.batch.Put(append([]byte(tb.prefix), key...), value) -} - -func (tb *tableBatch) Write() error { - return tb.batch.Write() -} - -func (tb *tableBatch) ValueSize() int { - return tb.batch.ValueSize() -} - -func (tb *tableBatch) Reset() { - tb.batch.Reset() -} diff --git a/shyftdb/interface.go b/shyftdb/interface.go deleted file mode 100644 index fe361cc791..0000000000 --- a/shyftdb/interface.go +++ /dev/null @@ -1,31 +0,0 @@ - -package shyftdb - -// Code using batches should try to add this much data to the batch. -// The value was determined empirically. -const IdealBatchSize = 100 * 1024 - -// Putter wraps the database write operation supported by both batches and regular databases. -type Putter interface { - Put(key []byte, value []byte) error -} - -// Database wraps all database operations. All methods are safe for concurrent use. -type Database interface { - Putter - Get(key []byte) ([]byte, error) - Has(key []byte) (bool, error) - Delete(key []byte) error - Close() - NewBatch() Batch -} - -// Batch is a write-only database that commits changes to its host database -// when Write is called. Batch cannot be used concurrently. -type Batch interface { - Putter - ValueSize() int // amount of data in the batch - Write() error - // Reset resets the batch for reuse - Reset() -} diff --git a/shyftdb/shyft_database_util.go b/shyftdb/shyft_database_util.go index af3475d4d6..c821f50423 100644 --- a/shyftdb/shyft_database_util.go +++ b/shyftdb/shyft_database_util.go @@ -12,6 +12,7 @@ import ( "log" _ "github.com/lib/pq" + "reflect" ) //SBlock type @@ -98,13 +99,13 @@ type SendAndReceive struct { } //WriteBlock writes to block info to sql db - func WriteBlock(sqldb *sql.DB, block *types.Block, receipts []*types.Receipt) error { +func WriteBlock(sqldb *sql.DB, block *types.Block, receipts []*types.Receipt) error { //Need to create field in postgres db isContract : True || False //Need to fix nonce numeric issue (attempt to reproduce and record) //Need to update tx To Field where null with Contract Address //Need to update account table with that Contract Address //Need to update AccountNonce and Balance of Tx To field || Contract Address - + WriteMinerRewards(sqldb,block) coinbase := block.Header().Coinbase.String() number := block.Header().Number.String() gasUsed := block.Header().GasUsed @@ -418,19 +419,44 @@ func WriteBalanceHelper(sqldb *sql.DB, tx *types.Transaction) (SendAndReceive, s return sendAndReceiveData, balanceReceiver, balanceSender, accountNonceReceiver, accountNonceSender } -//func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { -// var totalGas big.Int -// //var txs []string -// -// fmt.Println("this is BLOCK.UNCLE", block.Uncles()) -// fmt.Println("this is BLOCK UNCLE HASH", block.UncleHash().String()) -// fmt.Println("this is BLOCK.TRANSACTIONS", block.Transactions()) -// fmt.Println("this is BLOCK TOTAL GAS", block.GasUsed()) -// -// for _, tx := range block.Transactions() { -// totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) -// } -//} +func WriteMinerRewards(sqldb *sql.DB, block *types.Block) { + minerAddr := block.Coinbase().String() + fmt.Println("\n\n\t\t", minerAddr, "\n\n") + // Calculate the total gas used in the block + var totalGas big.Int + for _, tx := range block.Transactions() { + totalGas.Add(&totalGas, new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))) + } + + //TODO: Calculate Block Reward + // totalReward := totalGas.Add(&totalGas, MINER_REWARD) + totalReward := totalGas + totalRewardString := totalReward.String() + + // check if addr exists and update + var minerBalance string + addrExistsStatement := `SELECT balance from accounts WHERE addr = ($1)` + err := sqldb.QueryRow(addrExistsStatement, minerAddr).Scan(&minerBalance) + + // Create addr or update existing balance + if err == sql.ErrNoRows { + // Addr does not exist, thus create new entry + createAddrSqlStatement := `INSERT INTO accounts(addr, balance, txCountAccount) VALUES(($1), ($2), ($3)) RETURNING addr` + _, insertErr := sqldb.Exec(createAddrSqlStatement, minerAddr, totalRewardString, 0) + if insertErr != nil { + panic(insertErr) + } + } else if err != nil { + // Something went wrong panic + panic(err) + } else { + // Addr exists, update existing balance + var newBalance big.Int + newBalance.Add(addrInfo.balance, totalReward) + updateSQLStatement := `UPDATE accounts SET balance = ($1), txCountAccount = ($2) WHERE addr = ($3)` + _, updateErr := sqldb.Exec(updateSQLStatement, ) + } +} // @NOTE: This function is extremely complex and requires heavy testing and knowdlege of edge cases: // uncle blocks, account balance updates based on reorgs, diverges that get dropped. diff --git a/simulations/sendTransactions.js b/simulations/sendTransactions.js index b819a8338a..95ce1b71c5 100644 --- a/simulations/sendTransactions.js +++ b/simulations/sendTransactions.js @@ -1,29 +1,25 @@ -var firstAccount = web3.eth.accounts[0] -var secondAccount = web3.eth.accounts[1] -var thirdAccount = web3.eth.accounts[2] - for (var i = 0; i < 1; i++) { console.log('\t\t' + (i + 1) + ' - Transactions') web3.eth.sendTransaction({ - from: web3.eth.accounts[1], + from: web3.eth.accounts[3], to: web3.eth.accounts[2], value: 5, gas: 50000, gasPrice: 20 }); - // web3.eth.sendTransaction({ - // from: web3.eth.accounts[0], - // to: web3.eth.accounts[2], - // value: 291, - // gas: 50000, - // gasPrice: 20 - // }); - // - // web3.eth.sendTransaction({ - // from: web3.eth.accounts[0], - // to: web3.eth.accounts[1], - // value: 53039, - // gas: 50000, - // gasPrice: 20 - // }); + web3.eth.sendTransaction({ + from: web3.eth.accounts[3], + to: web3.eth.accounts[2], + value: 291, + gas: 50000, + gasPrice: 20 + }); + + web3.eth.sendTransaction({ + from: web3.eth.accounts[3], + to: web3.eth.accounts[1], + value: 53039, + gas: 50000, + gasPrice: 20 + }); } \ No newline at end of file