From 6654544ca7245fa0f501057a94e74c8e331af2b0 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 19 Dec 2013 13:38:09 -0500 Subject: [PATCH 01/18] Added initial files --- blocks.py | 185 +++++++++++++++++++++++++++++++++++++++++ parser.py | 5 ++ processblock.py | 214 ++++++++++++++++++++++++++++++++++++++++++++++++ rlp.py | 50 +++++++++++ transactions.py | 30 +++++++ 5 files changed, 484 insertions(+) create mode 100644 blocks.py create mode 100644 parser.py create mode 100644 processblock.py create mode 100644 rlp.py create mode 100644 transactions.py diff --git a/blocks.py b/blocks.py new file mode 100644 index 0000000000..ee519ca8cf --- /dev/null +++ b/blocks.py @@ -0,0 +1,185 @@ +from pybitcointools import * +import rlp +import re +from transactions import Transaction + +class Block(): + def __init__(self,data=None): + if not data: + return + if re.match('^[0-9a-fA-F]*$',data): + data = data.decode('hex') + header, tree_node_list, transaction_list, sibling_list = rlp.decode(data) + h = rlp.decode(header) + self.prevhash = encode(h[0],16,64) + self.coinbase = encode(h[1],16,40) + self.balance_root = encode(h[2],256,32) + self.contract_root = encode(h[3],256,32) + self.difficulty = h[4] + self.timestamp = h[5] + transactions_root = encode(h[6],256,32) + siblings_root = encode(h[7],256,32) + self.nonce = h[8] + self.datastore = {} + for nd in rlp.decode(tree_node_list): + ndk = bin_sha256(nd) + self.datastore[ndk] = rlp.decode(nd) + self.transactions = [Transaction(x) for x in rlp.decode(transaction_list)] + self.siblings = [rlp.decode(x) for x in rlp.decode(sibling_list)] + # Verifications + if self.balance_root != '' and self.balance_root not in self.datastore: + raise Exception("Balance Merkle root not found!") + if self.contract_root != '' and self.contract_root not in self.datastore: + raise Exception("Contract Merkle root not found!") + if bin_sha256(transaction_list) != transactions_root: + raise Exception("Transaction list root hash does not match!") + if bin_sha256(sibling_list) != sibling_root: + raise Exception("Transaction list root hash does not match!") + for sibling in self.siblings: + if sibling[0] != self.prevhash: + raise Exception("Sibling's parent is not my parent!") + + + hexalpha = '0123456789abcdef' + + def get_updated_state(self,node,key,value): + curnode = self.datastore.get(node,None) + # Insertion case + if value != 0 and value != '': + # Base case + if key == '': + return value + # Inserting into an empty trie + if not curnode: + newnode = [ key, value ] + k = sha256(rlp.encode(newnode)) + self.datastore[k] = newnode + return k + elif len(curnode) == 2: + # Inserting into a (k,v), same key + if key == curnode[0]: + newnode = [ key, value ] + k = sha256(rlp.encode(newnode)) + self.datastore[k] = newnode + return k + # Inserting into a (k,v), different key + else: + i = 0 + while key[:i] == curnode[0][:i]: i += 1 + k1 = self.get_updated_state(None,curnode[0][i:],curnode[1]) + k2 = self.get_updated_state(None,key[i:],value) + newnode3 = [ None ] * 16 + newnode3[ord(curnode[0][0])] = k1 + newnode3[ord(key[0])] = k2 + k3 = sha256(rlp.encode(newnode3)) + self.datastore[k3] = newnode3 + # No prefix sharing + if i == 1: + return k3 + # Prefix sharing + else: + newnode4 = [ key[:i-1], k3 ] + k4 = sha256(rlp.encode(newnode4)) + self.datastore[k4] = newnode4 + return k4 + else: + # inserting into a 16-array + newnode1 = self.get_updated_state(curnode[ord(key[0])],key[1:],value) + newnode2 = [ curnode[i] for i in range(16) ] + newnode2[ord(key[0])] = newnode1 + return newnode2 + # Deletion case + else: + # Base case + if key == '': + return None + # Deleting from a (k,v); obvious + if len(curnode) == 2: + if key == curnode[0]: return None + else: return node + else: + k1 = self.get_updated_state(curnode[ord(key[0])],key[1:],value) + newnode = [ curnode[i] for i in range(16) ] + newnode[ord(key[0])] = k1 + totalnodes = sum([ 1 if newnode2[i] else 0 for i in range(16) ]) + if totalnodes == 0: + raise Exception("Can't delete from two to zero! Error! Waahmbulance!") + elif totalnodes == 1: + # If only have one node left, we revert to (key, value) + node_index = [i for i in range(16) if newnode2[i]][0] + node2 = self.datastore[curnode[node_index]] + if len(node2) == 2: + # If it's a (key, value), we just prepend to the key + newnode = [ chr(node_index) + node2[0], node2[1] ] + else: + # Otherwise, we just make a single-char (key, value) pair + newnode = [ chr(node_index), curnode[node_index] ] + k2 = sha256(rlp.encode(newnode)) + self.datastore[k2] = newnode + return k2 + + + def update_balance(self,address,value): + # Use out internal representation for the key + key = ''.join([chr(hexalpha.find(x)) for x in address.encode('hex')]) + self.balance_root = self.get_updated_state(self.balance_root,key,value) + + def update_contract_state(self,address,index,value): + # Use out internal representation for the key + key = ''.join([chr(hexalpha.find(x)) for x in (address+index).encode('hex')]) + self.contract_root = self.get_updated_state(self.contract_root,key,value) + + def get_state_value(self,node,key): + if key == '': + return node + if not curnode: + return None + curnode = self.datastore.get(node,None) + return self.get_state_value(curnode[ord(key[0])],key[1:]) + + def get_balance(self,address): + # Use out internal representation for the key + key = ''.join([chr(hexalpha.find(x)) for x in (address).encode('hex')]) + return self.get_state_value(self.balance_root,key) + + def get_contract_state(self,address,index): + # Use out internal representation for the key + key = ''.join([chr(hexalpha.find(x)) for x in (address+index).encode('hex')]) + return self.get_state_value(self.contract_root,key) + + def get_state_size(self,node): + if node is None: return 0 + curnode = self.datastore.get(node,None) + if not curnode: return 0 + elif len(curnode) == 2: + return self.get_state_size(curnode[1]) + else: + total = 0 + for i in range(16): total += self.get_state_size(curnode[i]) + return total + + def get_contract_size(self,address): + # Use out internal representation for the key + key = ''.join([chr(hexalpha.find(x)) for x in (address).encode('hex')]) + return self.get_state_size(self.get_state_value(self.contract_root,key)) + + def serialize(self): + nodes = {} + def process(node): + if node is None: return + curnode = self.datastore.get(node,None) + if curnode: + index = sha256(rlp.encode(curnode)) + nodes[index] = curnode + if len(node) == 2: + process(curnode[1]) + elif len(node) == 16: + for i in range(16): process(curnode[i]) + process(self.balance_root) + process(self.contract_root) + tree_nodes = [nodes[x] for x in nodes] + nodelist = rlp.encode(tree_nodes) + txlist = rlp.encode([x.serialize() for x in self.transactions]) + siblinglist = rlp.encode(self.siblings) + header = rlp.encode([self.prevhash, self.coinbase, self.balance_root, self.contract_root, self.difficulty, self.timestamp, bin_sha256(txlist), bin_sha256(siblinglist]) + return rlp.encode([header, nodelist, txlist, siblinglist]) diff --git a/parser.py b/parser.py new file mode 100644 index 0000000000..00823aeaa3 --- /dev/null +++ b/parser.py @@ -0,0 +1,5 @@ +import rlp + +def parse(inp): + if inp[0] == '\x00': + return { "type": "transaction", "data": rlp.parse( diff --git a/processblock.py b/processblock.py new file mode 100644 index 0000000000..20d7edb4da --- /dev/null +++ b/processblock.py @@ -0,0 +1,214 @@ +from transactions import Transaction +from blocks import Block + +scriptcode_map = { + 0x00: 'STOP', + 0x10: 'ADD', + 0x11: 'SUB', + 0x12: 'MUL', + 0x13: 'DIV', + 0x14: 'SDIV', + 0x15: 'MOD', + 0x16: 'SMOD', + 0x17: 'EXP', + 0x20: 'LT', + 0x21: 'LE', + 0x22: 'GT', + 0x23: 'GE', + 0x24: 'EQ', + 0x25: 'NEG', + 0x26: 'NOT', + 0x30: 'SHA256', + 0x31: 'RIPEMD-160', + 0x32: 'ECMUL', + 0x33: 'ECADD', + 0x34: 'SIGN', + 0x35: 'RECOVER', + 0x40: 'COPY', + 0x41: 'STORE', + 0x42: 'LD', + 0x43: 'SET', + 0x50: 'JMP', + 0x51: 'JMPI', + 0x52: 'IND', + 0x60: 'EXTRO', + 0x61: 'BALANCE', + 0x70: 'MKTX', + 0x71: 'RAWTX', + 0x80: 'DATA', + 0x81: 'DATAN', + 0x90: 'MYADDRESS' +} + +fees = { + 'stepfee': 2**60 * 8192, + 'txfee': 2**60 * 524288, + 'memoryfee': 2**60 * 262144, + 'memory_adjust_fee': 2**60 * 65536 +} + +def eval_contract(block,tx): + address = tx.to + # Initialize registers + reg = [0] * 256 + reg[0] = decode(tx.from,16) + reg[1] = decode(tx.to,16) + reg[2] = tx.value + reg[3] = tx.fee + index = 0 + stepcounter = 0 + def monop(code,f): + reg[code[2]] = f(reg[code[1]]) + def binop(code,f): + reg[code[3]] = f(reg[code[1]],reg[code[2]]) + while 1: + # Calculate fee + totalfee = 0 + stepcounter += 1 + if stepcounter > 16: + totalfee += fees.get("stepfee") + val_at_index = decode(block.get_contract_state(address,encode(index,256,32)),256) + code = [ int(val_at_index / 256**i) % 256 for i in range(6) ] + c = scriptcode_map[code[0]] + if c == 'STORE': + existing = block.get_contract_state(address,code[2]) + if reg[code[1]] != 0: fee += fees["MEMORYFEE"] + if existing: fee -= fees["MEMORYFEE"] + contractbalance = block.get_balance(address) + # If we can't pay the fee... + if fee > contractbalance: + return state + # Otherwise, pay it + block.set_balance(address,contractbalance - fee) + + if c == 'STOP': + break + elif c == 'ADD': + reg[code[3]] = (reg[code[1]] + reg[code[2]]) % 2**256 + elif c == 'MUL': + reg[code[3]] = (reg[code[1]] * reg[code[2]]) % 2**256 + elif c == 'SUB': + reg[code[3]] = (reg[code[1]] + 2**256 - reg[code[2]]) % 2**256 + elif c == 'DIV': + reg[code[3]] = int(reg[code[1]] / reg[code[2]]) + elif c == 'SDIV': + sign = 1 + sign *= (1 if reg[code[1]] < 2**255 else -1) + sign *= (1 if reg[code[2]] < 2**255 else -1) + x = reg[code[1]] if reg[code[1]] < 2**255 else 2**256 - reg[code[1]] + y = reg[code[2]] if reg[code[2]] < 2**255 else 2**256 - reg[code[2]] + z = int(x/y) + reg[code[3]] = z if sign = 1 else 2**256 - z + elif code == 'MOD': + reg[code[3]] = reg[code[1]] % reg[code[2]] + elif code == 'SMOD': + sign = 1 + sign *= (1 if reg[code[1]] < 2**255 else -1) + sign *= (1 if reg[code[2]] < 2**255 else -1) + x = reg[code[1]] if reg[code[1]] < 2**255 else 2**256 - reg[code[1]] + y = reg[code[2]] if reg[code[2]] < 2**255 else 2**256 - reg[code[2]] + z = x%y + reg[code[3]] = z if sign = 1 else 2**256 - z + elif code == 'EXP': + reg[code[3]] = pow(reg[code[1]],reg[code[2]],2**256) + elif code == 'NEG': + reg[code[2]] = 2**256 - reg[code[1]] + elif code == 'LT': + reg[code[3]] = 1 if reg[code[1]] < reg[code[2]] else 0 + elif code == 'LE': + reg[code[3]] = 1 if reg[code[1]] <= reg[code[2]] else 0 + elif code == 'GT': + reg[code[3]] = 1 if reg[code[1]] > reg[code[2]] else 0 + elif code == 'GE': + reg[code[3]] = 1 if reg[code[1]] >= reg[code[2]] else 0 + elif code == 'EQ': + reg[code[3]] = 1 if reg[code[1]] == reg[code[2]] else 0 + elif code == 'NOT': + reg[code[2]] = 1 if reg[code[1]] == 0 else 0 + elif code == 'SHA256': + inp = encode(reg[code[1]],256,32) + reg[code[2]] = decode(hashlib.sha256(inp).digest(),256) + elif code == 'RIPEMD-160': + inp = encode(reg[code[1]],256,32) + reg[code[2]] = decode(hashlib.new('ripemd160',inp).digest(),256) + elif code == 'ECMUL': + pt = (reg[code[1]],reg[code[2]]) + # Point at infinity + if pt[0] == 0 and pt[1] == 0: + reg[code[4]], reg[code[5]] = 0,0 + # Point not on curve, coerce to infinity + elif (pt[0] ** 3 + 7 - pt[1] ** 2) % N != 0: + reg[code[4]], reg[code[5]] = 0,0 + # Legitimate point + else: + pt2 = base10_multiply(pt,reg[code[3]]) + reg[code[4]], reg[code[5]] = pt2[0], pt2[1] + elif code == 'ECADD': + pt1 = (reg[code[1]],reg[code[2]]) + pt2 = (reg[code[3]],reg[code[4]]) + if (pt1[0] ** 3 + 7 - pt1[1] ** 2) % N != 0: + reg[code[5]], reg[code[6]] = 0,0 + elif (pt2[0] ** 3 + 7 - pt2[1] ** 2) % N != 0: + reg[code[5]], reg[code[6]] = 0,0 + else: + pt3 = base10_add(pt1,pt2) + reg[code[5]], reg[code[6]] = pt3[0], pt3[1] + elif code == 'SIGN': + reg[code[3]], reg[code[4]], reg[code[5]] = ecdsa_raw_sign(reg[code[1]],reg[code[2]]) + elif code == 'RECOVER': + pt = ecdsa_raw_recover((reg[code[2]],reg[code[3]],reg[code[4]]),reg[code[1]]) + reg[code[5]] = pt[0] + reg[code[6]] = pt[1] + elif code == 'COPY': + reg[code[2]] = reg[code[1]] + elif code == 'STORE': + block.update_contract_state(address,encode(reg[code[2]],256,32),reg[code[1]]) + elif code == 'LD': + reg[code[2]] = block.get_contract_state(address,encode(reg[code[1]],256,32)) + elif code == 'SET': + reg[code[1]] = (code[2] + 256 * code[3] + 65536 * code[4] + 16777216 * code[5]) * 2**code[6] % 2**256 + elif code == 'JMP': + index = code[1] + elif code == 'JMPI': + if reg[code[1]]: index = code[2] + elif code == 'JMPX': + index = reg[code[1]] + elif code == 'JMPIX': + if reg[code[1]]: index = reg[code[2]] + elif code == 'IND': + reg[code[1]] = index + elif code == 'EXTRO': + reg[code[3]] = ethdb.get(ethdb.get(state.contract,enc160(reg[code[1]])),enc256(reg[code[2]])) + elif code == 'BALANCE': + reg[code[2]] = ethdb.get(state.balance,enc160(reg[code[1]])) + elif code == 'MKTX': + ntx = { + "ins": [ tx["receiver"] ], + "oindex": 0, + "sender": tx["receiver"], + "receiver": reg[code[1]], + "value": reg[code[2]], + "fee": reg[code[3]], + "data": [] + } + for i in range(reg[code[4]]): + ntx["data"].append(ethdb.get(contract,(reg[code[5]]+i) % 2**256)) + fee += ntx["fee"] + if fee > contractbalance: + return state + state.txs.append(tx) + elif code == 'DATA': + reg[code[2]] = tx["data"][code[1]] + elif code == 'DATAX': + reg[code[2]] = tx["data"][ethdb.get(contract,enc256(code[1]))] + elif code == 'DATAN': + reg[code[2]] = len(tx["data"]) + elif code == 'MYADDRESS': + reg[code[1]] = tx["receiver"] + elif code == 'SUICIDE': + sz = ethdb.size(contract) + fee -= sz * state.fees["MEMORYFEE"] + contract = None + state.balance = ethdb.set(state.balance,tx["receiver"],contractbalance - fee) + state.contract = contract + return state diff --git a/rlp.py b/rlp.py new file mode 100644 index 0000000000..3b2f16c15a --- /dev/null +++ b/rlp.py @@ -0,0 +1,50 @@ +def binary_length(n): + if n == 0: return 0 + else: return 1 + binary_length(n / 256) + +def to_binary_array(n,L=None): + if L is None: L = binary_length(n) + if n == 0: return [] + else: + x = to_binary_array(n / 256) + x.append(n % 256) + return x + +def to_binary(n,L=None): return ''.join([chr(x) for x in to_binary_array(n,L)]) + +def from_binary(b): + if len(b) == 0: return 0 + else: return ord(from_binary(b[:-1])) * 256 + b[-1] + +def num_to_var_int(n): + if n < 253: s = chr(n) + else if n < 2**16: s = [253] + list(reversed(to_binary_array(n,2))) + else if n < 2**32: s = [254] + list(reversed(to_binary_array(n,4))) + else if n < 2**64: s = [255] + list(reversed(to_binary_array(n,8))) + else raise Exception("number too big") + return ''.join([chr(x) for x in s]) + +def decode(s): + o = [] + index = 0 + def read_var_int(): + si = ord(s[index]) + index += 1 + if si < 253: return s[index - 1] + elif si == 253: read = 2 + elif si == 254: read = 4 + elif si == 255: read = 8 + index += read + return from_binary(s[index-read:index]) + while index < len(s): + L = read_var_int() + o.append(s[index:index+L]) + return o + +def encode(s): + if isinstance(s,(int,long)): return encode(to_binary(s)) + if isinstance(s,str): return num_to_var_int(len(s))+s + else: + x = ''.join([encode(x) for x in s]) + return num_to_var_int(len(s))+s + diff --git a/transactions.py b/transactions.py new file mode 100644 index 0000000000..d9b26b8f82 --- /dev/null +++ b/transactions.py @@ -0,0 +1,30 @@ +from pybitcointools import * +import rlp +import re + +class Transaction(): + def __init__(*args): + + def lpad(inp,L): return '\x00' * max(0,L - len(inp)) + inp + + def parse(self,data): + if re.match('^[0-9a-fA-F]*$',data): + data = data.decode('hex') + o = rlp.unparse(data) + self.to = lpad(o[0],20) + self.value = decode(o[1],256) + self.fee = decode(o[2],256) + self.data = rlp.unparse(o[-3]) + self.sig = o[-4] + rawhash = sha256(rlp.encode([self.to,self.value,self.fee,self.data])) + v,r,s = ord(self.sig[0]), decode(self.sig[1:33],256), decode(self.sig[33:],256) + self.from = hash160(ecdsa_raw_recover(rawhash,(v,r,s))) + def sign(self,key): + rawhash = sha256(rlp.parse([self.to,self.value,self.fee,self.data])) + v,r,s = ecdsa_raw_sign(rawhash,args[5]) + self.sig = chr(v)+encode(r,256,32)+encode(s,256,32) + self.from = hash160(privtopub(args[5])) + def serialize(self): + return rlp.parse([self.to, self.value, self.fee, self.data, self.sig]).encode('hex') + def hash(self): + return bin_sha256(self.serialize()) From d712f93e668f6c82d81cf8f631c76f992205dd9e Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 19 Dec 2013 21:26:52 -0500 Subject: [PATCH 02/18] Commit 2 --- processblock.py | 82 ++++++++++++++++++++++++++++--------------------- transactions.py | 8 +++++ 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/processblock.py b/processblock.py index 20d7edb4da..0c8455b117 100644 --- a/processblock.py +++ b/processblock.py @@ -43,10 +43,26 @@ scriptcode_map = { fees = { 'stepfee': 2**60 * 8192, 'txfee': 2**60 * 524288, - 'memoryfee': 2**60 * 262144, - 'memory_adjust_fee': 2**60 * 65536 + 'memoryfee': 2**60 * 262144 } +def eval_tx(block): + tx = block.transactions[0] + oldbalance = block.get_balance(tx.from) + debit = tx.value + tx.fee + if tx.to == '': + debit += fees['memoryfee'] * len(filter(lambda x:x > 0,tx.data)) + if oldbalance < debit: + return + block.update_balance(tx.from,oldbalance - debit) + if tx.to == '': + pass #todo: continue here + +def mk_contract(block,tx): + cdata = tx.data + # todo: continue here + + def eval_contract(block,tx): address = tx.to # Initialize registers @@ -168,47 +184,43 @@ def eval_contract(block,tx): elif code == 'SET': reg[code[1]] = (code[2] + 256 * code[3] + 65536 * code[4] + 16777216 * code[5]) * 2**code[6] % 2**256 elif code == 'JMP': - index = code[1] - elif code == 'JMPI': - if reg[code[1]]: index = code[2] - elif code == 'JMPX': index = reg[code[1]] - elif code == 'JMPIX': + elif code == 'JMPI': if reg[code[1]]: index = reg[code[2]] elif code == 'IND': reg[code[1]] = index elif code == 'EXTRO': - reg[code[3]] = ethdb.get(ethdb.get(state.contract,enc160(reg[code[1]])),enc256(reg[code[2]])) + address = encode(reg[code[1]] % 2**160,256,20) + field = encode(reg[code[2]] + reg[code[3]] = block.get_contract_state(address,field) elif code == 'BALANCE': - reg[code[2]] = ethdb.get(state.balance,enc160(reg[code[1]])) + address = encode(reg[code[1]] % 2**160,256,20) + reg[code[2]] = block.get_balance(address) elif code == 'MKTX': - ntx = { - "ins": [ tx["receiver"] ], - "oindex": 0, - "sender": tx["receiver"], - "receiver": reg[code[1]], - "value": reg[code[2]], - "fee": reg[code[3]], - "data": [] - } - for i in range(reg[code[4]]): - ntx["data"].append(ethdb.get(contract,(reg[code[5]]+i) % 2**256)) - fee += ntx["fee"] - if fee > contractbalance: - return state - state.txs.append(tx) + to = encode(reg[code[1]],256,32) + value = reg[code[2]] + fee = reg[code[3]] + if (value + fee) > block.get_balance(address): + pass + else: + datan = reg[code[4]] + data = [] + for i in range(datan): + ind = encode((reg[code[5]] + i) % 2**256,256,32) + data.append(block.get_contract_state(address,ind)) + tx = Transaction(to,value,fee,data) + tx.from = address + block.transactions.append(tx) elif code == 'DATA': - reg[code[2]] = tx["data"][code[1]] - elif code == 'DATAX': - reg[code[2]] = tx["data"][ethdb.get(contract,enc256(code[1]))] + reg[code[2]] = tx.data[reg[code[1]]] elif code == 'DATAN': - reg[code[2]] = len(tx["data"]) + reg[code[1]] = len(tx.data) elif code == 'MYADDRESS': - reg[code[1]] = tx["receiver"] + reg[code[1]] = address elif code == 'SUICIDE': - sz = ethdb.size(contract) - fee -= sz * state.fees["MEMORYFEE"] - contract = None - state.balance = ethdb.set(state.balance,tx["receiver"],contractbalance - fee) - state.contract = contract - return state + sz = block.get_contract_size(address) + negfee = sz * fees["memoryfee"] + toaddress = encode(reg[code[1]],256,32) + block.update_balance(toaddress,block.get_balance(toaddress) + negfee) + block.update_contract(address,0) + break diff --git a/transactions.py b/transactions.py index d9b26b8f82..0754ea1084 100644 --- a/transactions.py +++ b/transactions.py @@ -4,6 +4,14 @@ import re class Transaction(): def __init__(*args): + if len(args) == 2: + self.parse(args[1]) + else: + self.to = args[1] + self.value = args[2] + self.fee = args[3] + self.data = args[4] + if len(args) > 5: self.sig = args[5] def lpad(inp,L): return '\x00' * max(0,L - len(inp)) + inp From 541c7ee4465234b75b8943654f5d33dede43af94 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 19 Dec 2013 21:28:47 -0500 Subject: [PATCH 03/18] Commit 2 --- processblock.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/processblock.py b/processblock.py index 0c8455b117..ef3405b5f5 100644 --- a/processblock.py +++ b/processblock.py @@ -47,7 +47,7 @@ fees = { } def eval_tx(block): - tx = block.transactions[0] + tx = block.transactions.pop(0) oldbalance = block.get_balance(tx.from) debit = tx.value + tx.fee if tx.to == '': @@ -56,7 +56,9 @@ def eval_tx(block): return block.update_balance(tx.from,oldbalance - debit) if tx.to == '': - pass #todo: continue here + mk_contract(block,tx) #todo: continue here + else: + block.update_balance(tx.to,block.get_balance(tx.to) + tx.value) def mk_contract(block,tx): cdata = tx.data From ec9c2aff854a941d747c32bb0556ba1a85efd8b9 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 19 Dec 2013 21:30:38 -0500 Subject: [PATCH 04/18] Commit 3 --- processblock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/processblock.py b/processblock.py index ef3405b5f5..1be89ef022 100644 --- a/processblock.py +++ b/processblock.py @@ -59,6 +59,8 @@ def eval_tx(block): mk_contract(block,tx) #todo: continue here else: block.update_balance(tx.to,block.get_balance(tx.to) + tx.value) + if block.get_contract(tx.to) != 0: + eval_contract(block,tx) def mk_contract(block,tx): cdata = tx.data From e476fb19bc0d55fb120772c175e438e21a0cefb1 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 21 Dec 2013 21:20:30 -0500 Subject: [PATCH 05/18] Added independent RLP and trie files --- processblock.py | 4 +- rlp.py | 42 +++++++------ trie.py | 155 ++++++++++++++++++++++++++++++++++++++++++++++++ trietest.py | 25 ++++++++ 4 files changed, 206 insertions(+), 20 deletions(-) create mode 100644 trie.py create mode 100644 trietest.py diff --git a/processblock.py b/processblock.py index 1be89ef022..4fb7700890 100644 --- a/processblock.py +++ b/processblock.py @@ -11,13 +11,13 @@ scriptcode_map = { 0x15: 'MOD', 0x16: 'SMOD', 0x17: 'EXP', + 0x18: 'NEG', 0x20: 'LT', 0x21: 'LE', 0x22: 'GT', 0x23: 'GE', 0x24: 'EQ', - 0x25: 'NEG', - 0x26: 'NOT', + 0x25: 'NOT', 0x30: 'SHA256', 0x31: 'RIPEMD-160', 0x32: 'ECMUL', diff --git a/rlp.py b/rlp.py index 3b2f16c15a..708c770d3e 100644 --- a/rlp.py +++ b/rlp.py @@ -14,37 +14,43 @@ def to_binary(n,L=None): return ''.join([chr(x) for x in to_binary_array(n,L)]) def from_binary(b): if len(b) == 0: return 0 - else: return ord(from_binary(b[:-1])) * 256 + b[-1] + else: return from_binary(b[:-1]) * 256 + ord(b[-1]) def num_to_var_int(n): - if n < 253: s = chr(n) - else if n < 2**16: s = [253] + list(reversed(to_binary_array(n,2))) - else if n < 2**32: s = [254] + list(reversed(to_binary_array(n,4))) - else if n < 2**64: s = [255] + list(reversed(to_binary_array(n,8))) - else raise Exception("number too big") + if n < 253: s = [n] + elif n < 2**16: s = [253] + list(to_binary_array(n,2)) + elif n < 2**32: s = [254] + list(to_binary_array(n,4)) + elif n < 2**64: s = [255] + list(to_binary_array(n,8)) + else: raise Exception("number too big") return ''.join([chr(x) for x in s]) -def decode(s): +def __decode(s): o = [] - index = 0 + index = [0] def read_var_int(): - si = ord(s[index]) - index += 1 - if si < 253: return s[index - 1] + si = ord(s[index[0]]) + index[0] += 1 + if si < 253: return si elif si == 253: read = 2 elif si == 254: read = 4 elif si == 255: read = 8 - index += read - return from_binary(s[index-read:index]) - while index < len(s): + index[0] += read + return from_binary(s[index[0]-read:index[0]]) + while index[0] < len(s): + tp = s[index[0]] + index[0] += 1 L = read_var_int() - o.append(s[index:index+L]) + item = s[index[0]:index[0]+L] + if tp == '\x00': o.append(item) + else: o.append(__decode(item)) + index[0] += L return o +def decode(s): return __decode(s)[0] + def encode(s): if isinstance(s,(int,long)): return encode(to_binary(s)) - if isinstance(s,str): return num_to_var_int(len(s))+s + if isinstance(s,str): return '\x00'+num_to_var_int(len(s))+s else: x = ''.join([encode(x) for x in s]) - return num_to_var_int(len(s))+s - + return '\x01'+num_to_var_int(len(x))+x diff --git a/trie.py b/trie.py new file mode 100644 index 0000000000..2d435744ee --- /dev/null +++ b/trie.py @@ -0,0 +1,155 @@ +import leveldb +import rlp +import hashlib + +def sha256(x): return hashlib.sha256(x).digest() + +class DB(): + def __init__(self,dbfile): self.db = leveldb.LevelDB(dbfile) + def get(self,key): + try: return self.db.Get(key) + except: return '' + def put(self,key,value): return self.db.Put(key,value) + def delete(self,key): return self.db.Delete(key) + +class Trie(): + def __init__(self,db,root='',debug=False): + self.root = root + self.db = DB(db) + self.debug = debug + + def __encode_key(self,key): + term = 1 if key[-1] == 16 else 0 + oddlen = (len(key) - term) % 2 + prefix = ('0' if oddlen else '') + main = ''.join(['0123456789abcdef'[x] for x in key[:len(key)-term]]) + return chr(2 * term + oddlen) + (prefix+main).decode('hex') + + def __decode_key(self,key): + o = ['0123456789abcdef'.find(x) for x in key[1:].encode('hex')] + if key[0] == '\x01' or key[0] == '\x03': o = o[1:] + if key[0] == '\x02' or key[0] == '\x03': o.append(16) + return o + + def __get_state(self,node,key): + if debug: print 'nk',node.encode('hex'),key + if len(key) == 0 or not node: + return node + curnode = rlp.decode(self.db.get(node)) + if debug: print 'cn', curnode + if not curnode: + raise Exception("node not found in database") + elif len(curnode) == 2: + (k2,v2) = curnode + k2 = self.__decode_key(k2) + if len(key) >= len(k2) and k2 == key[:len(k2)]: + return self.__get_state(v2,key[len(k2):]) + else: + return '' + elif len(curnode) == 17: + return self.__get_state(curnode[key[0]],key[1:]) + + def __put(self,node): + rlpnode = rlp.encode(node) + h = sha256(rlpnode) + self.db.put(h,rlpnode) + return h + + def __update_state(self,node,key,value): + if value != '': return self.__insert_state(node,key,value) + else: return self.__delete_state(node,key) + + def __insert_state(self,node,key,value): + if debug: print 'ink', node.encode('hex'), key + if len(key) == 0: + return value + else: + if not node: + newnode = [ self.__encode_key(key), value ] + return self.__put(newnode) + curnode = rlp.decode(self.db.get(node)) + if debug: print 'icn', curnode + if not curnode: + raise Exception("node not found in database") + if len(curnode) == 2: + (k2, v2) = curnode + k2 = self.__decode_key(k2) + if key == k2: + newnode = [ self.__encode_key(key), value ] + return self.__put(newnode) + else: + i = 0 + while key[:i+1] == k2[:i+1] and i < len(k2): i += 1 + if i == len(k2): + newhash3 = self.__insert_state(v2,key[i:],value) + else: + newnode1 = self.__insert_state('',key[i+1:],value) + newnode2 = self.__insert_state('',k2[i+1:],v2) + newnode3 = [ '' ] * 17 + newnode3[key[i]] = newnode1 + newnode3[k2[i]] = newnode2 + newhash3 = self.__put(newnode3) + if i == 0: + return newhash3 + else: + newnode4 = [ self.__encode_key(key[:i]), newhash3 ] + return self.__put(newnode4) + else: + newnode = [ curnode[i] for i in range(17) ] + newnode[key[0]] = self.__insert_state(curnode[key[0]],key[1:],value) + return self.__put(newnode) + + def __delete_state(self,node,key): + if debug: print 'dnk', node.encode('hex'), key + if len(key) == 0 or not node: + return '' + else: + curnode = rlp.decode(self.db.get(node)) + if not curnode: + raise Exception("node not found in database") + if debug: print 'dcn', curnode + if len(curnode) == 2: + (k2, v2) = curnode + k2 = self.__decode_key(k2) + if key == k2: + return '' + elif key[:len(k2)] == k2: + newhash = self.__delete_state(v2,key[len(k2):]) + childnode = rlp.decode(self.db.get(newhash)) + if len(childnode) == 2: + newkey = k2 + self.__decode_key(childnode[0]) + newnode = [ self.__encode_key(newkey), childnode[1] ] + else: + newnode = [ curnode[0], newhash ] + return self.__put(newnode) + else: return node + else: + newnode = [ curnode[i] for i in range(17) ] + newnode[key[0]] = self.__delete_state(newnode[key[0]],key[1:]) + onlynode = -1 + for i in range(17): + if newnode[i]: + if onlynode == -1: onlynode = i + else: onlynode = -2 + if onlynode >= 0: + childnode = rlp.decode(self.db.get(newnode[onlynode])) + if not childnode: + raise Exception("?????") + if len(childnode) == 17: + newnode2 = [ key[0], newnode[onlynode] ] + elif len(childnode) == 2: + newkey = [onlynode] + self.__decode_key(childnode[0]) + newnode2 = [ self.__encode_key(newkey), childnode[1] ] + else: + newnode2 = newnode + return self.__put(newnode2) + + def get(self,key): + key2 = ['0123456789abcdef'.find(x) for x in key.encode('hex')] + [16] + return self.__get_state(self.root,key2) + + def update(self,key,value): + if not isinstance(key,str) or not isinstance(value,str): + raise Exception("Key and value must be strings") + key2 = ['0123456789abcdef'.find(x) for x in key.encode('hex')] + [16] + self.root = self.__update_state(self.root,key2,value) diff --git a/trietest.py b/trietest.py new file mode 100644 index 0000000000..538c99a0f4 --- /dev/null +++ b/trietest.py @@ -0,0 +1,25 @@ +from trie import Trie +import random + +def genkey(): + L = random.randrange(30) + if random.randrange(5) == 0: return '' + return ''.join([random.choice('1234579qetyiasdfghjklzxcvbnm') for x in range(L)]) + +t = Trie('/tmp/'+genkey()) + +def trie_test(): + o = {} + for i in range(60): + key, value = genkey(), genkey() + if value: print "setting key: '"+key+"', value: '"+value+"'" + else: print "deleting key: '"+key+"'" + o[key] = value + t.update(key,value) + for k in o.keys(): + v1 = o[k] + v2 = t.get(k) + print v1,v2 + if v1 != v2: raise Exception("incorrect!") + + From bce41be4f77eacdb260b9514b1471cd4058e4f93 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 21 Dec 2013 21:38:16 -0500 Subject: [PATCH 06/18] whoops small changes --- trie.py | 12 ++++++------ trietest.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/trie.py b/trie.py index 2d435744ee..d7eb900c45 100644 --- a/trie.py +++ b/trie.py @@ -32,11 +32,11 @@ class Trie(): return o def __get_state(self,node,key): - if debug: print 'nk',node.encode('hex'),key + if self.debug: print 'nk',node.encode('hex'),key if len(key) == 0 or not node: return node curnode = rlp.decode(self.db.get(node)) - if debug: print 'cn', curnode + if self.debug: print 'cn', curnode if not curnode: raise Exception("node not found in database") elif len(curnode) == 2: @@ -60,7 +60,7 @@ class Trie(): else: return self.__delete_state(node,key) def __insert_state(self,node,key,value): - if debug: print 'ink', node.encode('hex'), key + if self.debug: print 'ink', node.encode('hex'), key if len(key) == 0: return value else: @@ -68,7 +68,7 @@ class Trie(): newnode = [ self.__encode_key(key), value ] return self.__put(newnode) curnode = rlp.decode(self.db.get(node)) - if debug: print 'icn', curnode + if self.debug: print 'icn', curnode if not curnode: raise Exception("node not found in database") if len(curnode) == 2: @@ -100,14 +100,14 @@ class Trie(): return self.__put(newnode) def __delete_state(self,node,key): - if debug: print 'dnk', node.encode('hex'), key + if self.debug: print 'dnk', node.encode('hex'), key if len(key) == 0 or not node: return '' else: curnode = rlp.decode(self.db.get(node)) if not curnode: raise Exception("node not found in database") - if debug: print 'dcn', curnode + if self.debug: print 'dcn', curnode if len(curnode) == 2: (k2, v2) = curnode k2 = self.__decode_key(k2) diff --git a/trietest.py b/trietest.py index 538c99a0f4..d427b1e4f9 100644 --- a/trietest.py +++ b/trietest.py @@ -22,4 +22,4 @@ def trie_test(): print v1,v2 if v1 != v2: raise Exception("incorrect!") - +trie_test() From 0d729e121938b623857544c6c95bc95fc02d38a7 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 22 Dec 2013 16:06:51 -0500 Subject: [PATCH 07/18] Finished everything except the miner, PoW validator and testing --- blocks.py | 263 ++++++++++++++++++++---------------------------- processblock.py | 195 +++++++++++++++++++++++------------ rlp.py | 1 + transactions.py | 43 ++++---- trie.py | 26 ++++- 5 files changed, 286 insertions(+), 242 deletions(-) diff --git a/blocks.py b/blocks.py index ee519ca8cf..de1df78633 100644 --- a/blocks.py +++ b/blocks.py @@ -2,35 +2,38 @@ from pybitcointools import * import rlp import re from transactions import Transaction +from trie import Trie class Block(): def __init__(self,data=None): + if not data: return + if re.match('^[0-9a-fA-F]*$',data): data = data.decode('hex') - header, tree_node_list, transaction_list, sibling_list = rlp.decode(data) - h = rlp.decode(header) - self.prevhash = encode(h[0],16,64) - self.coinbase = encode(h[1],16,40) - self.balance_root = encode(h[2],256,32) - self.contract_root = encode(h[3],256,32) - self.difficulty = h[4] - self.timestamp = h[5] - transactions_root = encode(h[6],256,32) - siblings_root = encode(h[7],256,32) - self.nonce = h[8] - self.datastore = {} - for nd in rlp.decode(tree_node_list): - ndk = bin_sha256(nd) - self.datastore[ndk] = rlp.decode(nd) - self.transactions = [Transaction(x) for x in rlp.decode(transaction_list)] - self.siblings = [rlp.decode(x) for x in rlp.decode(sibling_list)] + + header, transaction_list, self.siblings = rlp.decode(data) + [ number, + self.prevhash, + self.siblings_root, + self.coinbase, + state_root, + self.transactions_root, + diff, + timestamp, + nonce, + self.extra ] = header + self.number = decode(number,256) + self.difficulty = decode(difficulty,256) + self.timestamp = decode(timestamp,256) + self.nonce = decode(nonce,256) + self.transactions = [Transaction(x) for x in transaction_list)] + self.state = Trie('statedb',state_root) + # Verifications - if self.balance_root != '' and self.balance_root not in self.datastore: - raise Exception("Balance Merkle root not found!") - if self.contract_root != '' and self.contract_root not in self.datastore: - raise Exception("Contract Merkle root not found!") + if self.state.root != '' and self.state.__get_state(self.state.root,[]) == '': + raise Exception("State Merkle root not found in database!") if bin_sha256(transaction_list) != transactions_root: raise Exception("Transaction list root hash does not match!") if bin_sha256(sibling_list) != sibling_root: @@ -38,148 +41,96 @@ class Block(): for sibling in self.siblings: if sibling[0] != self.prevhash: raise Exception("Sibling's parent is not my parent!") + # TODO: check POW - - hexalpha = '0123456789abcdef' - - def get_updated_state(self,node,key,value): - curnode = self.datastore.get(node,None) - # Insertion case - if value != 0 and value != '': - # Base case - if key == '': - return value - # Inserting into an empty trie - if not curnode: - newnode = [ key, value ] - k = sha256(rlp.encode(newnode)) - self.datastore[k] = newnode - return k - elif len(curnode) == 2: - # Inserting into a (k,v), same key - if key == curnode[0]: - newnode = [ key, value ] - k = sha256(rlp.encode(newnode)) - self.datastore[k] = newnode - return k - # Inserting into a (k,v), different key - else: - i = 0 - while key[:i] == curnode[0][:i]: i += 1 - k1 = self.get_updated_state(None,curnode[0][i:],curnode[1]) - k2 = self.get_updated_state(None,key[i:],value) - newnode3 = [ None ] * 16 - newnode3[ord(curnode[0][0])] = k1 - newnode3[ord(key[0])] = k2 - k3 = sha256(rlp.encode(newnode3)) - self.datastore[k3] = newnode3 - # No prefix sharing - if i == 1: - return k3 - # Prefix sharing - else: - newnode4 = [ key[:i-1], k3 ] - k4 = sha256(rlp.encode(newnode4)) - self.datastore[k4] = newnode4 - return k4 - else: - # inserting into a 16-array - newnode1 = self.get_updated_state(curnode[ord(key[0])],key[1:],value) - newnode2 = [ curnode[i] for i in range(16) ] - newnode2[ord(key[0])] = newnode1 - return newnode2 - # Deletion case + def send(self,tx): + # Subtract value and fee from sender account and increment nonce if applicable + sender_state = rlp.decode(self.state.get(tx.from)) + if not sender_state: + return False + sender_value = decode(sender_state[1],256) + if value + fee > sender_value: + return False + sender_state[1] = encode(sender_value - value - fee,256) + # Nonce applies only to key-based addresses + if decode(sender_state[0],256) == 0: + if decode(sender_state[2],256) != tx.nonce: + return False + sender_state[2] = encode(tx.nonce + 1,256) + self.state.update(tx.from,sender_state) + # Add money to receiver + if tx.to > '': + receiver_state = rlp.decode(self.state.get(tx.to)) or ['', '', ''] + receiver_state[1] = encode(decode(receiver_state[1],256) + value,256) + self.state.update(tx.to,receiver_state) + # Create a new contract else: - # Base case - if key == '': - return None - # Deleting from a (k,v); obvious - if len(curnode) == 2: - if key == curnode[0]: return None - else: return node - else: - k1 = self.get_updated_state(curnode[ord(key[0])],key[1:],value) - newnode = [ curnode[i] for i in range(16) ] - newnode[ord(key[0])] = k1 - totalnodes = sum([ 1 if newnode2[i] else 0 for i in range(16) ]) - if totalnodes == 0: - raise Exception("Can't delete from two to zero! Error! Waahmbulance!") - elif totalnodes == 1: - # If only have one node left, we revert to (key, value) - node_index = [i for i in range(16) if newnode2[i]][0] - node2 = self.datastore[curnode[node_index]] - if len(node2) == 2: - # If it's a (key, value), we just prepend to the key - newnode = [ chr(node_index) + node2[0], node2[1] ] - else: - # Otherwise, we just make a single-char (key, value) pair - newnode = [ chr(node_index), curnode[node_index] ] - k2 = sha256(rlp.encode(newnode)) - self.datastore[k2] = newnode - return k2 + addr = tx.hash()[:20] + contract = block.get_contract(addr) + if contract.root != '': return False + for i in range(len(tx.data)): + contract.update(encode(i,256,32),tx.data[i]) + block.update_contract(addr) + # Pay fee to miner + miner_state = rlp_decode(self.state.get(self.coinbase)) or ['','',''] + miner_state[1] = encode(decode(miner_state[1],256) + fee,256) + self.state.update(self.coinbase,miner_state) + return True + def pay_fee(self,address,fee,tominer=True): + # Subtract fee from sender + sender_state = rlp.decode(self.state.get(address)) + if not sender_state: + return False + sender_value = decode(sender_state[1],256) + if sender_value < fee: + return False + sender_state[1] = encode(sender_value - fee,256) + self.state.update(address,sender_state) + # Pay fee to miner + if tominer: + miner_state = rlp.decode(self.state.get(self.coinbase)) or ['','',''] + miner_state[1] = encode(decode(miner_state[1],256) + fee,256) + self.state.update(self.coinbase,miner_state) + return True - def update_balance(self,address,value): - # Use out internal representation for the key - key = ''.join([chr(hexalpha.find(x)) for x in address.encode('hex')]) - self.balance_root = self.get_updated_state(self.balance_root,key,value) - - def update_contract_state(self,address,index,value): - # Use out internal representation for the key - key = ''.join([chr(hexalpha.find(x)) for x in (address+index).encode('hex')]) - self.contract_root = self.get_updated_state(self.contract_root,key,value) - - def get_state_value(self,node,key): - if key == '': - return node - if not curnode: - return None - curnode = self.datastore.get(node,None) - return self.get_state_value(curnode[ord(key[0])],key[1:]) + def get_nonce(self,address): + state = rlp.decode(self.state.get(address)) + if not state or decode(state[0],256) == 0: return False + return decode(state[2],256) def get_balance(self,address): - # Use out internal representation for the key - key = ''.join([chr(hexalpha.find(x)) for x in (address).encode('hex')]) - return self.get_state_value(self.balance_root,key) + state = rlp.decode(self.state.get(address)) + return decode(state[1] || '',256) - def get_contract_state(self,address,index): - # Use out internal representation for the key - key = ''.join([chr(hexalpha.find(x)) for x in (address+index).encode('hex')]) - return self.get_state_value(self.contract_root,key) + # Making updates to the object obtained from this method will do nothing. You need + # to call update_contract to finalize the changes. + def get_contract(self,address): + state = rlp.decode(self.state.get(address)) + if not state or decode(state[0],256) == 0: return False + return Trie('statedb',state[2]) - def get_state_size(self,node): - if node is None: return 0 - curnode = self.datastore.get(node,None) - if not curnode: return 0 - elif len(curnode) == 2: - return self.get_state_size(curnode[1]) - else: - total = 0 - for i in range(16): total += self.get_state_size(curnode[i]) - return total + def update_contract(self,address,contract): + state = rlp.decode(self.state.get(address)) + if not state or decode(state[0],256) == 0: return False + state[2] = contract.root + self.state.update(address,state) - def get_contract_size(self,address): - # Use out internal representation for the key - key = ''.join([chr(hexalpha.find(x)) for x in (address).encode('hex')]) - return self.get_state_size(self.get_state_value(self.contract_root,key)) - + # Serialization method; should act as perfect inverse function of the constructor + # assuming no verification failures def serialize(self): - nodes = {} - def process(node): - if node is None: return - curnode = self.datastore.get(node,None) - if curnode: - index = sha256(rlp.encode(curnode)) - nodes[index] = curnode - if len(node) == 2: - process(curnode[1]) - elif len(node) == 16: - for i in range(16): process(curnode[i]) - process(self.balance_root) - process(self.contract_root) - tree_nodes = [nodes[x] for x in nodes] - nodelist = rlp.encode(tree_nodes) - txlist = rlp.encode([x.serialize() for x in self.transactions]) - siblinglist = rlp.encode(self.siblings) - header = rlp.encode([self.prevhash, self.coinbase, self.balance_root, self.contract_root, self.difficulty, self.timestamp, bin_sha256(txlist), bin_sha256(siblinglist]) - return rlp.encode([header, nodelist, txlist, siblinglist]) + txlist = [x.serialize() for x in self.transactions] + header = [ encode(self.number,256), + self.prevhash, + bin_sha256(rlp.encode(self.siblings)), + self.coinbase, + self.state.root, + bin_sha256(rlp.encode(self.txlist)), + encode(self.difficulty,256), + encode(self.timestamp,256), + encode(self.nonce,256), + self.extra ] + return rlp.encode([header, txlist, self.siblings ]) + + def hash(self): + return bin_sha256(self.serialize()) diff --git a/processblock.py b/processblock.py index 4fb7700890..6f3eff24f1 100644 --- a/processblock.py +++ b/processblock.py @@ -1,5 +1,6 @@ from transactions import Transaction from blocks import Block +import time scriptcode_map = { 0x00: 'STOP', @@ -22,11 +23,11 @@ scriptcode_map = { 0x31: 'RIPEMD-160', 0x32: 'ECMUL', 0x33: 'ECADD', - 0x34: 'SIGN', - 0x35: 'RECOVER', + 0x34: 'ECSIGN', + 0x35: 'ECRECOVER', 0x40: 'COPY', 0x41: 'STORE', - 0x42: 'LD', + 0x42: 'LOAD', 0x43: 'SET', 0x50: 'JMP', 0x51: 'JMPI', @@ -37,73 +38,125 @@ scriptcode_map = { 0x71: 'RAWTX', 0x80: 'DATA', 0x81: 'DATAN', - 0x90: 'MYADDRESS' + 0x90: 'MYADDRESS', + 0x91: 'BLKHASH', + 0xff: 'SUICIDE' } -fees = { - 'stepfee': 2**60 * 8192, +params = { + 'stepfee': 2**60 * 4096, 'txfee': 2**60 * 524288, - 'memoryfee': 2**60 * 262144 + 'memoryfee': 2**60 * 262144, + 'datafee': 2**60 * 16384, + 'cryptofee': 2**60 * 65536, + 'extrofee': 2**60 * 65536, + 'blocktime': 60, + 'period_1_reward': 2**80 * 1024, + 'period_1_duration': 57600, + 'period_2_reward': 2**80 * 512, + 'period_2_duration': 57600, + 'period_3_reward': 2**80 * 256, + 'period_3_duration': 57600, + 'period_4_reward': 2**80 * 128 } -def eval_tx(block): - tx = block.transactions.pop(0) - oldbalance = block.get_balance(tx.from) - debit = tx.value + tx.fee - if tx.to == '': - debit += fees['memoryfee'] * len(filter(lambda x:x > 0,tx.data)) - if oldbalance < debit: - return - block.update_balance(tx.from,oldbalance - debit) - if tx.to == '': - mk_contract(block,tx) #todo: continue here +def eval(block,transactions,timestamp,coinbase): + h = block.hash() + # Process all transactions + while len(transactions) > 0: + tx = transactions.pop(0) + fee = params['txfee'] + len(tx.data) * params['datafee'] + if tx.to = '\x00'*20: + fee += len(tx.data) * params['memoryfee'] + # Insufficient fee, do nothing + if fee > tx.fee: continue + # Too much data, do nothing + if len(data) > 256: continue + # Try to send the tx + if not block.send(tx): continue + # Evaluate contract if applicable + eval_contract(block,transactions,tx) + # Pay miner fee + miner_state = rlp_decode(self.state.get(self.coinbase)) or ['','',''] + miner_balance = decode(miner_state[1],256) + block.number += 1 + reward = 0 + if block.number < params['period_1_duration']: + reward = params['period_1_reward'] + elif block.number < params['period_2_duration']: + reward = params['period_2_reward'] + elif block.number < params['period_3_duration']: + reward = params['period_3_reward'] else: - block.update_balance(tx.to,block.get_balance(tx.to) + tx.value) - if block.get_contract(tx.to) != 0: - eval_contract(block,tx) + reward = params['period_4_reward'] + miner_balance += reward + for sibling in block.siblings: + sib_miner_state = rlp_decode(self.state.get(sibling[3])) + sib_miner_state[1] = encode(decode(sib_miner_state[1],256)+reward*7/8,256) + self.state.update(sibling[3],sib_miner_state) + miner_balance += reward/8 + miner_state[1] = encode(miner_balance,256) + self.state.update(self.coinbase,miner_state) + # Check timestamp + if timestamp < block.timestamp or timestamp > int(time.time()) + 3600: + raise new Exception("timestamp not in valid range!") + # Update difficulty + if timestamp >= block.timestamp + 42: + block.difficulty += int(block.difficulty / 1024) + else: + block.difficulty -= int(block.difficulty / 1024) + block.prevhash = h + block.coinbase = coinbase + block.transactions = [] + block.siblings = [] + return block -def mk_contract(block,tx): - cdata = tx.data - # todo: continue here - - -def eval_contract(block,tx): +def eval_contract(block,transaction_list,tx): address = tx.to # Initialize registers reg = [0] * 256 - reg[0] = decode(tx.from,16) - reg[1] = decode(tx.to,16) + reg[0] = decode(tx.from,256) + reg[1] = decode(tx.to,256) reg[2] = tx.value reg[3] = tx.fee index = 0 stepcounter = 0 - def monop(code,f): - reg[code[2]] = f(reg[code[1]]) - def binop(code,f): - reg[code[3]] = f(reg[code[1]],reg[code[2]]) + contract = block.get_contract(address) + if not contract: + return while 1: + # Convert the data item into a code piece + val_at_index = decode(contract.get(encode(index,256,32)),256) + code = [ int(val_at_index / (256**i)) % 256 for i in range(6) ] + # Invalid code instruction or STOP code stops execution sans fee + if val_at_index >= 256**6 or code[0] == 'STOP': + break # Calculate fee - totalfee = 0 + minerfee = 0 + nullfee = 0 stepcounter += 1 if stepcounter > 16: - totalfee += fees.get("stepfee") - val_at_index = decode(block.get_contract_state(address,encode(index,256,32)),256) - code = [ int(val_at_index / 256**i) % 256 for i in range(6) ] + minerfee += params["stepfee"] c = scriptcode_map[code[0]] + if c in ['STORE','LOAD']: + minerfee += params["datafee"] + if c in ['EXTRO','BALANCE']: + minerfee += params["extrofee"] + if c in ['SHA256','RIPEMD-160','ECMUL','ECADD','ECSIGN','ECRECOVER']: + minerfee += params["cryptofee"] if c == 'STORE': existing = block.get_contract_state(address,code[2]) - if reg[code[1]] != 0: fee += fees["MEMORYFEE"] - if existing: fee -= fees["MEMORYFEE"] - contractbalance = block.get_balance(address) - # If we can't pay the fee... - if fee > contractbalance: - return state - # Otherwise, pay it - block.set_balance(address,contractbalance - fee) + if reg[code[1]] != 0: nullfee += params["memoryfee"] + if existing: nullfee -= params["memoryfee"] - if c == 'STOP': + # If we can't pay the fee, break, otherwise pay it + if block.get_balance(address) < minerfee + nullfee: break - elif c == 'ADD': + block.pay_fee(address,nullfee,False) + block.pay_fee(address,minerfee,True) + + # Evaluate operations + if c == 'ADD': reg[code[3]] = (reg[code[1]] + reg[code[2]]) % 2**256 elif c == 'MUL': reg[code[3]] = (reg[code[1]] * reg[code[2]]) % 2**256 @@ -166,25 +219,28 @@ def eval_contract(block,tx): elif code == 'ECADD': pt1 = (reg[code[1]],reg[code[2]]) pt2 = (reg[code[3]],reg[code[4]]) + # Invalid point 1 if (pt1[0] ** 3 + 7 - pt1[1] ** 2) % N != 0: reg[code[5]], reg[code[6]] = 0,0 + # Invalid point 2 elif (pt2[0] ** 3 + 7 - pt2[1] ** 2) % N != 0: reg[code[5]], reg[code[6]] = 0,0 + # Legitimate points else: pt3 = base10_add(pt1,pt2) reg[code[5]], reg[code[6]] = pt3[0], pt3[1] - elif code == 'SIGN': + elif code == 'ECSIGN': reg[code[3]], reg[code[4]], reg[code[5]] = ecdsa_raw_sign(reg[code[1]],reg[code[2]]) - elif code == 'RECOVER': + elif code == 'ECRECOVER': pt = ecdsa_raw_recover((reg[code[2]],reg[code[3]],reg[code[4]]),reg[code[1]]) reg[code[5]] = pt[0] reg[code[6]] = pt[1] elif code == 'COPY': reg[code[2]] = reg[code[1]] elif code == 'STORE': - block.update_contract_state(address,encode(reg[code[2]],256,32),reg[code[1]]) - elif code == 'LD': - reg[code[2]] = block.get_contract_state(address,encode(reg[code[1]],256,32)) + contract.update(encode(reg[code[2]],256,32),reg[code[1]]) + elif code == 'LOAD': + reg[code[2]] = contract.get(encode(reg[code[1]],256,32)) elif code == 'SET': reg[code[1]] = (code[2] + 256 * code[3] + 65536 * code[4] + 16777216 * code[5]) * 2**code[6] % 2**256 elif code == 'JMP': @@ -194,12 +250,18 @@ def eval_contract(block,tx): elif code == 'IND': reg[code[1]] = index elif code == 'EXTRO': - address = encode(reg[code[1]] % 2**160,256,20) - field = encode(reg[code[2]] - reg[code[3]] = block.get_contract_state(address,field) + if reg[code[1]] >= 2**160: + reg[code[3]] = 0 + else: + address = encode(reg[code[1]],256,20) + field = encode(reg[code[2]]) + reg[code[3]] = block.get_contract(address).get(field) elif code == 'BALANCE': - address = encode(reg[code[1]] % 2**160,256,20) - reg[code[2]] = block.get_balance(address) + if reg[code[1]] >= 2**160: + reg[code[2]] = 0 + else: + address = encode(reg[code[1]],256,20) + reg[code[2]] = block.get_balance(address) elif code == 'MKTX': to = encode(reg[code[1]],256,32) value = reg[code[2]] @@ -211,20 +273,23 @@ def eval_contract(block,tx): data = [] for i in range(datan): ind = encode((reg[code[5]] + i) % 2**256,256,32) - data.append(block.get_contract_state(address,ind)) - tx = Transaction(to,value,fee,data) + data.append(contract.get(ind)) + tx = Transaction(0,to,value,fee,data) tx.from = address - block.transactions.append(tx) + transaction_list.insert(0,tx) elif code == 'DATA': reg[code[2]] = tx.data[reg[code[1]]] elif code == 'DATAN': reg[code[1]] = len(tx.data) elif code == 'MYADDRESS': reg[code[1]] = address + elif code == 'BLKHASH': + reg[code[1]] = decode(block.hash()) elif code == 'SUICIDE': - sz = block.get_contract_size(address) - negfee = sz * fees["memoryfee"] - toaddress = encode(reg[code[1]],256,32) - block.update_balance(toaddress,block.get_balance(toaddress) + negfee) - block.update_contract(address,0) + sz = contract.get_size() + negfee = -sz * params["memoryfee"] + toaddress = encode(reg[code[1]],256,20) + block.pay_fee(roaddress,negfee,False) + contract.root = '' break + block.update_contract(address,contract) diff --git a/rlp.py b/rlp.py index 708c770d3e..7bc5db2668 100644 --- a/rlp.py +++ b/rlp.py @@ -25,6 +25,7 @@ def num_to_var_int(n): return ''.join([chr(x) for x in s]) def __decode(s): + if s == '': return None o = [] index = [0] def read_var_int(): diff --git a/transactions.py b/transactions.py index 0754ea1084..ef457190b1 100644 --- a/transactions.py +++ b/transactions.py @@ -7,32 +7,37 @@ class Transaction(): if len(args) == 2: self.parse(args[1]) else: - self.to = args[1] - self.value = args[2] - self.fee = args[3] - self.data = args[4] - if len(args) > 5: self.sig = args[5] - - def lpad(inp,L): return '\x00' * max(0,L - len(inp)) + inp + self.nonce = args[1] + self.to = args[2] + self.value = args[3] + self.fee = args[4] + self.data = args[5] def parse(self,data): if re.match('^[0-9a-fA-F]*$',data): data = data.decode('hex') o = rlp.unparse(data) - self.to = lpad(o[0],20) - self.value = decode(o[1],256) - self.fee = decode(o[2],256) - self.data = rlp.unparse(o[-3]) - self.sig = o[-4] - rawhash = sha256(rlp.encode([self.to,self.value,self.fee,self.data])) - v,r,s = ord(self.sig[0]), decode(self.sig[1:33],256), decode(self.sig[33:],256) - self.from = hash160(ecdsa_raw_recover(rawhash,(v,r,s))) + self.nonce = decode(o[0],256) + self.to = o[1] + self.value = decode(o[2],256) + self.fee = decode(o[3],256) + self.data = rlp.unparse(o[4]) + self.v = o[5] + self.r = o[6] + self.s = o[7] + rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) + self.from = hash160(ecdsa_raw_recover(rawhash,(self.v,self.r,self.s))) + def sign(self,key): rawhash = sha256(rlp.parse([self.to,self.value,self.fee,self.data])) - v,r,s = ecdsa_raw_sign(rawhash,args[5]) - self.sig = chr(v)+encode(r,256,32)+encode(s,256,32) - self.from = hash160(privtopub(args[5])) + self.v,self.r,self.s = ecdsa_raw_sign(rawhash,key) + self.from = hash160(privtopub(key)) + def serialize(self): - return rlp.parse([self.to, self.value, self.fee, self.data, self.sig]).encode('hex') + return rlp.parse([self.nonce, self.to, self.value, self.fee, self.data, self.v, self.r, self.s]) + + def hex_serialize(self): + return self.serialize().encode('hex') + def hash(self): return bin_sha256(self.serialize()) diff --git a/trie.py b/trie.py index d7eb900c45..0b7ff1365d 100644 --- a/trie.py +++ b/trie.py @@ -12,11 +12,15 @@ class DB(): def put(self,key,value): return self.db.Put(key,value) def delete(self,key): return self.db.Delete(key) +databases = {} + class Trie(): - def __init__(self,db,root='',debug=False): + def __init__(self,dbfile,root='',debug=False): self.root = root - self.db = DB(db) self.debug = debug + if dbfile not in databases: + databases[dbfile] = DB(dbfile) + self.db = databases[dbfile] def __encode_key(self,key): term = 1 if key[-1] == 16 else 0 @@ -144,10 +148,28 @@ class Trie(): newnode2 = newnode return self.__put(newnode2) + def __get_size(self,node): + if not node: return 0 + curnode = self.db.get(node) + if not curnode: + raise Exception("node not found in database") + if len(curnode) == 2: + key = self.__decode_key(curnode[0]) + if key[-1] == 16: return 1 + else: return self.__get_size(curnode[1]) + elif len(curnode) == 17: + total = 0 + for i in range(16): + total += self.__get_size(curnode[i]) + if curnode[16]: total += 1 + return total + def get(self,key): key2 = ['0123456789abcdef'.find(x) for x in key.encode('hex')] + [16] return self.__get_state(self.root,key2) + def get_size(self): return self.__get_size(self.root) + def update(self,key,value): if not isinstance(key,str) or not isinstance(value,str): raise Exception("Key and value must be strings") From a3af9af23ede211b9fe933fc437074393ce36dfc Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Wed, 25 Dec 2013 08:26:43 -0500 Subject: [PATCH 08/18] Redid RLP --- blocks.py | 17 ++++----- processblock.py | 27 +++++++------- rlp.py | 95 +++++++++++++++++++++++++++++++------------------ 3 files changed, 83 insertions(+), 56 deletions(-) diff --git a/blocks.py b/blocks.py index de1df78633..097b71cb0f 100644 --- a/blocks.py +++ b/blocks.py @@ -13,10 +13,10 @@ class Block(): if re.match('^[0-9a-fA-F]*$',data): data = data.decode('hex') - header, transaction_list, self.siblings = rlp.decode(data) + header, transaction_list, self.uncles = rlp.decode(data) [ number, self.prevhash, - self.siblings_root, + self.uncles_root, self.coinbase, state_root, self.transactions_root, @@ -36,11 +36,8 @@ class Block(): raise Exception("State Merkle root not found in database!") if bin_sha256(transaction_list) != transactions_root: raise Exception("Transaction list root hash does not match!") - if bin_sha256(sibling_list) != sibling_root: - raise Exception("Transaction list root hash does not match!") - for sibling in self.siblings: - if sibling[0] != self.prevhash: - raise Exception("Sibling's parent is not my parent!") + if bin_sha256(uncle_list) != uncle_root: + raise Exception("Uncle root hash does not match!") # TODO: check POW def send(self,tx): @@ -65,7 +62,7 @@ class Block(): self.state.update(tx.to,receiver_state) # Create a new contract else: - addr = tx.hash()[:20] + addr = tx.hash()[-20:] contract = block.get_contract(addr) if contract.root != '': return False for i in range(len(tx.data)): @@ -122,7 +119,7 @@ class Block(): txlist = [x.serialize() for x in self.transactions] header = [ encode(self.number,256), self.prevhash, - bin_sha256(rlp.encode(self.siblings)), + bin_sha256(rlp.encode(self.uncles)), self.coinbase, self.state.root, bin_sha256(rlp.encode(self.txlist)), @@ -130,7 +127,7 @@ class Block(): encode(self.timestamp,256), encode(self.nonce,256), self.extra ] - return rlp.encode([header, txlist, self.siblings ]) + return rlp.encode([header, txlist, self.uncles ]) def hash(self): return bin_sha256(self.serialize()) diff --git a/processblock.py b/processblock.py index 6f3eff24f1..3bbfa61636 100644 --- a/processblock.py +++ b/processblock.py @@ -44,12 +44,13 @@ scriptcode_map = { } params = { - 'stepfee': 2**60 * 4096, - 'txfee': 2**60 * 524288, - 'memoryfee': 2**60 * 262144, - 'datafee': 2**60 * 16384, - 'cryptofee': 2**60 * 65536, - 'extrofee': 2**60 * 65536, + 'stepfee': 2**64 / 64, + 'txfee': 2**64, + 'newcontractfee': 2**64, + 'memoryfee': 2**64 / 4, + 'datafee': 2**64 / 16, + 'cryptofee': 2**64 / 16, + 'extrofee': 2**64 / 16, 'blocktime': 60, 'period_1_reward': 2**80 * 1024, 'period_1_duration': 57600, @@ -65,9 +66,11 @@ def eval(block,transactions,timestamp,coinbase): # Process all transactions while len(transactions) > 0: tx = transactions.pop(0) - fee = params['txfee'] + len(tx.data) * params['datafee'] + fee = 0 if tx.to = '\x00'*20: - fee += len(tx.data) * params['memoryfee'] + fee += params['newcontractfee'] + len(tx.data) * params['memoryfee'] + else: + fee += params['txfee'] # Insufficient fee, do nothing if fee > tx.fee: continue # Too much data, do nothing @@ -90,10 +93,10 @@ def eval(block,transactions,timestamp,coinbase): else: reward = params['period_4_reward'] miner_balance += reward - for sibling in block.siblings: - sib_miner_state = rlp_decode(self.state.get(sibling[3])) + for uncle in block.uncles: + sib_miner_state = rlp_decode(self.state.get(uncle[3])) sib_miner_state[1] = encode(decode(sib_miner_state[1],256)+reward*7/8,256) - self.state.update(sibling[3],sib_miner_state) + self.state.update(uncle[3],sib_miner_state) miner_balance += reward/8 miner_state[1] = encode(miner_balance,256) self.state.update(self.coinbase,miner_state) @@ -108,7 +111,7 @@ def eval(block,transactions,timestamp,coinbase): block.prevhash = h block.coinbase = coinbase block.transactions = [] - block.siblings = [] + block.uncles = [] return block def eval_contract(block,transaction_list,tx): diff --git a/rlp.py b/rlp.py index 7bc5db2668..03272b6c7c 100644 --- a/rlp.py +++ b/rlp.py @@ -16,42 +16,69 @@ def from_binary(b): if len(b) == 0: return 0 else: return from_binary(b[:-1]) * 256 + ord(b[-1]) -def num_to_var_int(n): - if n < 253: s = [n] - elif n < 2**16: s = [253] + list(to_binary_array(n,2)) - elif n < 2**32: s = [254] + list(to_binary_array(n,4)) - elif n < 2**64: s = [255] + list(to_binary_array(n,8)) - else: raise Exception("number too big") - return ''.join([chr(x) for x in s]) - -def __decode(s): - if s == '': return None - o = [] - index = [0] - def read_var_int(): - si = ord(s[index[0]]) - index[0] += 1 - if si < 253: return si - elif si == 253: read = 2 - elif si == 254: read = 4 - elif si == 255: read = 8 - index[0] += read - return from_binary(s[index[0]-read:index[0]]) - while index[0] < len(s): - tp = s[index[0]] - index[0] += 1 - L = read_var_int() - item = s[index[0]:index[0]+L] - if tp == '\x00': o.append(item) - else: o.append(__decode(item)) - index[0] += L - return o +def __decode(s,pos=0): + if s == '': + return (None, 0) + else: + fchar = ord(s[pos]) + if fchar <= 24: + return (ord(s[pos]), pos+1) + elif fchar < 56: + b = ord(s[pos]) - 24 + return (from_binary(s[pos+1:pos+1+b]), pos+1+b) + elif fchar < 64: + b = ord(s[pos]) - 55 + b2 = from_binary(s[pos+1:pos+1+b]) + return (from_binary(s[pos+1+b:pos+1+b+b2]), pos+1+b+b2) + elif fchar < 120: + b = ord(s[pos]) - 64 + return (s[pos+1:pos+1+b], pos+1+b) + elif fchar < 128: + b = ord(s[pos]) - 119 + b2 = from_binary(s[pos+1:pos+1+b]) + return (s[pos+1+b:pos+1+b+b2], pos+1+b+b2) + elif fchar < 184: + b = ord(s[pos]) - 128 + o, pos = [], pos+1 + for i in range(b): + obj, pos = __decode(s,pos) + o.append(obj) + return (o,pos) + elif fchar < 192: + b = ord(s[pos]) - 183 + b2 = from_binary(s[pos+1:pos+1+b]) + o, pos = [], pos+1+b + for i in range(b): + obj, pos = __decode(s,pos) + o.append(obj) + return (o,pos) + else: + raise Exception("byte not supported: "+fchar) def decode(s): return __decode(s)[0] def encode(s): - if isinstance(s,(int,long)): return encode(to_binary(s)) - if isinstance(s,str): return '\x00'+num_to_var_int(len(s))+s + if isinstance(s,(int,long)): + if s <= 24: + return chr(s) + elif s <= 2**256: + b = to_binary(s) + return chr(len(b) + 24) + b + else: + b = to_binary(s) + b2 = to_binary(len(b)) + return chr(len(b2) + 55) + b2 + b + elif isinstance(s,str): + if len(s) < 56: + return chr(len(s) + 64) + s + else: + b2 = to_binary(len(s)) + return chr(len(b2) + 119) + b2 + s + elif isinstance(s,list): + if len(s) < 56: + return chr(len(s) + 128) + ''.join([encode(x) for x in s]) + else: + b2 = to_binary(len(s)) + return chr(len(b2) + 183) + b2 + ''.join([encode(x) for x in s]) else: - x = ''.join([encode(x) for x in s]) - return '\x01'+num_to_var_int(len(x))+x + raise Exception("Encoding for "+s+" not yet implemented") From c489ff0cbbe2dfd0049fd3e793c1c49ccb388545 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 26 Dec 2013 16:17:18 -0500 Subject: [PATCH 09/18] Got the block updating, serializing and deserializing to work at least without contracts --- blocks.py | 95 ++++++++++++++---------------------------- manager.py | 89 +++++++++++++++++++++++++++++++++++++++ processblock.py | 108 ++++++++++++++++++++++++++++++++++-------------- rlp.py | 12 +++--- transactions.py | 20 +++++---- trie.py | 39 +++++++++++++++++ 6 files changed, 254 insertions(+), 109 deletions(-) create mode 100644 manager.py diff --git a/blocks.py b/blocks.py index 097b71cb0f..c9e94c2098 100644 --- a/blocks.py +++ b/blocks.py @@ -3,6 +3,7 @@ import rlp import re from transactions import Transaction from trie import Trie +import sys class Block(): def __init__(self,data=None): @@ -14,102 +15,68 @@ class Block(): data = data.decode('hex') header, transaction_list, self.uncles = rlp.decode(data) - [ number, + [ self.number, self.prevhash, self.uncles_root, self.coinbase, state_root, self.transactions_root, - diff, - timestamp, - nonce, + self.difficulty, + self.timestamp, + self.nonce, self.extra ] = header - self.number = decode(number,256) - self.difficulty = decode(difficulty,256) - self.timestamp = decode(timestamp,256) - self.nonce = decode(nonce,256) - self.transactions = [Transaction(x) for x in transaction_list)] + self.transactions = [Transaction(x) for x in transaction_list] self.state = Trie('statedb',state_root) + self.reward = 0 # Verifications - if self.state.root != '' and self.state.__get_state(self.state.root,[]) == '': + if self.state.root != '' and self.state.db.get(self.state.root) == '': raise Exception("State Merkle root not found in database!") - if bin_sha256(transaction_list) != transactions_root: + if bin_sha256(rlp.encode(transaction_list)) != self.transactions_root: raise Exception("Transaction list root hash does not match!") - if bin_sha256(uncle_list) != uncle_root: + if bin_sha256(rlp.encode(self.uncles)) != self.uncles_root: raise Exception("Uncle root hash does not match!") # TODO: check POW - def send(self,tx): - # Subtract value and fee from sender account and increment nonce if applicable - sender_state = rlp.decode(self.state.get(tx.from)) - if not sender_state: - return False - sender_value = decode(sender_state[1],256) - if value + fee > sender_value: - return False - sender_state[1] = encode(sender_value - value - fee,256) - # Nonce applies only to key-based addresses - if decode(sender_state[0],256) == 0: - if decode(sender_state[2],256) != tx.nonce: - return False - sender_state[2] = encode(tx.nonce + 1,256) - self.state.update(tx.from,sender_state) - # Add money to receiver - if tx.to > '': - receiver_state = rlp.decode(self.state.get(tx.to)) or ['', '', ''] - receiver_state[1] = encode(decode(receiver_state[1],256) + value,256) - self.state.update(tx.to,receiver_state) - # Create a new contract - else: - addr = tx.hash()[-20:] - contract = block.get_contract(addr) - if contract.root != '': return False - for i in range(len(tx.data)): - contract.update(encode(i,256,32),tx.data[i]) - block.update_contract(addr) - # Pay fee to miner - miner_state = rlp_decode(self.state.get(self.coinbase)) or ['','',''] - miner_state[1] = encode(decode(miner_state[1],256) + fee,256) - self.state.update(self.coinbase,miner_state) - return True - def pay_fee(self,address,fee,tominer=True): # Subtract fee from sender sender_state = rlp.decode(self.state.get(address)) - if not sender_state: + if not sender_state or sender_state[1] < fee: return False - sender_value = decode(sender_state[1],256) - if sender_value < fee: - return False - sender_state[1] = encode(sender_value - fee,256) + sender_state[1] -= fee self.state.update(address,sender_state) # Pay fee to miner if tominer: - miner_state = rlp.decode(self.state.get(self.coinbase)) or ['','',''] - miner_state[1] = encode(decode(miner_state[1],256) + fee,256) + miner_state = rlp.decode(self.state.get(self.coinbase)) or [0,0,0] + miner_state[1] += fee self.state.update(self.coinbase,miner_state) return True def get_nonce(self,address): state = rlp.decode(self.state.get(address)) - if not state or decode(state[0],256) == 0: return False - return decode(state[2],256) + if not state or state[0] == 0: return False + return state[2] def get_balance(self,address): state = rlp.decode(self.state.get(address)) - return decode(state[1] || '',256) + return state[1] if state else 0 + + def set_balance(self,address,balance): + state = rlp.decode(self.state.get(address)) or [0,0,0] + state[1] = balance + self.state.update(address,rlp.encode(state)) + # Making updates to the object obtained from this method will do nothing. You need # to call update_contract to finalize the changes. def get_contract(self,address): state = rlp.decode(self.state.get(address)) - if not state or decode(state[0],256) == 0: return False + if not state or state[0] == 0: return False return Trie('statedb',state[2]) def update_contract(self,address,contract): - state = rlp.decode(self.state.get(address)) - if not state or decode(state[0],256) == 0: return False + state = rlp.decode(self.state.get(address)) or [1,0,''] + if state[0] == 0: return False state[2] = contract.root self.state.update(address,state) @@ -117,15 +84,15 @@ class Block(): # assuming no verification failures def serialize(self): txlist = [x.serialize() for x in self.transactions] - header = [ encode(self.number,256), + header = [ self.number, self.prevhash, bin_sha256(rlp.encode(self.uncles)), self.coinbase, self.state.root, - bin_sha256(rlp.encode(self.txlist)), - encode(self.difficulty,256), - encode(self.timestamp,256), - encode(self.nonce,256), + bin_sha256(rlp.encode(txlist)), + self.difficulty, + self.timestamp, + self.nonce, self.extra ] return rlp.encode([header, txlist, self.uncles ]) diff --git a/manager.py b/manager.py new file mode 100644 index 0000000000..dbe0f37cdd --- /dev/null +++ b/manager.py @@ -0,0 +1,89 @@ +import rlp +import leveldb +from blocks import Block +from transactions import Transaction +import processblock +import hashlib +from pybitcointools import * + +txpool = {} + +genesis_header = [ + 0, + '', + bin_sha256(rlp.encode([])), + '', + '', + bin_sha256(rlp.encode([])), + 2**36, + 0, + 0, + '' +] + +genesis = [ genesis_header, [], [] ] + +mainblk = Block(rlp.encode(genesis)) + +db = leveldb.LevelDB("objects") + +def genaddr(seed): + priv = bin_sha256(seed) + addr = bin_sha256(privtopub(priv)[1:])[-20:] + return priv,addr + +# For testing +k1,a1 = genaddr("123") +k2,a2 = genaddr("456") + +def broadcast(obj): + pass + +def receive(obj): + d = rlp.decode(obj) + # Is transaction + if len(d) == 8: + tx = Transaction(obj) + if mainblk.get_balance(tx.sender) < tx.value + tx.fee: return + if mainblk.get_nonce(tx.sender) != tx.nonce: return + txpool[bin_sha256(blk)] = blk + broadcast(blk) + # Is message + elif len(d) == 2: + if d[0] == 'getobj': + try: return db.Get(d[1][0]) + except: + try: return mainblk.state.db.get(d[1][0]) + except: return None + elif d[0] == 'getbalance': + try: return mainblk.state.get_balance(d[1][0]) + except: return None + elif d[0] == 'getcontractroot': + try: return mainblk.state.get_contract(d[1][0]).root + except: return None + elif d[0] == 'getcontractsize': + try: return mainblk.state.get_contract(d[1][0]).get_size() + except: return None + elif d[0] == 'getcontractstate': + try: return mainblk.state.get_contract(d[1][0]).get(d[1][1]) + except: return None + # Is block + elif len(d) == 3: + blk = Block(obj) + p = block.prevhash + try: + parent = Block(db.Get(p)) + except: + return + uncles = block.uncles + for s in uncles: + try: + sib = db.Get(s) + except: + return + processblock.eval(parent,blk.transactions,blk.timestamp,blk.coinbase) + if parent.state.root != blk.state.root: return + if parent.difficulty != blk.difficulty: return + if parent.number != blk.number: return + db.Put(blk.hash(),blk.serialize()) + diff --git a/processblock.py b/processblock.py index 3bbfa61636..64a233ac35 100644 --- a/processblock.py +++ b/processblock.py @@ -1,6 +1,8 @@ from transactions import Transaction from blocks import Block import time +import sys +import rlp scriptcode_map = { 0x00: 'STOP', @@ -61,27 +63,64 @@ params = { 'period_4_reward': 2**80 * 128 } +def process_transactions(block,transactions): + while len(transactions) > 0: + tx = transactions.pop(0) + enc = (tx.value, tx.fee, tx.sender.encode('hex'), tx.to.encode('hex')) + sys.stderr.write("Attempting to send %d plus fee %d from %s to %s\n" % enc) + # Grab data about sender, recipient and miner + sdata = rlp.decode(block.state.get(tx.sender)) or [0,0,0] + tdata = rlp.decode(block.state.get(tx.to)) or [0,0,0] + # Calculate fee + if tx.to == '\x00'*20: + fee = params['newcontractfee'] + len(tx.data) * params['memoryfee'] + else: + fee = params['txfee'] + # Insufficient fee, do nothing + if fee > tx.fee: + sys.stderr.write("Insufficient fee\n") + continue + # Too much data, do nothing + if len(tx.data) > 256: + sys.stderr.write("Too many data items\n") + continue + if not sdata or sdata[1] < tx.value + tx.fee: + sys.stderr.write("Insufficient funds to send fee\n") + continue + elif tx.nonce != sdata[2] and sdata[0] == 0: + sys.stderr.write("Bad nonce\n") + continue + # Try to send the tx + if sdata[0] == 0: sdata[2] += 1 + sdata[1] -= (tx.value + tx.fee) + block.reward += tx.fee + if tx.to != '': + tdata[1] += tx.value + else: + addr = tx.hash()[-20:] + adata = rlp.decode(block.state.get(addr)) + if adata[2] != '': + sys.stderr.write("Contract already exists\n") + continue + block.state.update(addr,rlp.encode([1,tx.value,''])) + contract = block.get_contract(addr) + for i in range(len(tx.data)): + contract.update(encode(i,256,32),tx.data[i]) + block.update_contract(addr) + print sdata, tdata + block.state.update(tx.sender,rlp.encode(sdata)) + block.state.update(tx.to,rlp.encode(tdata)) + # Evaluate contract if applicable + if tdata[0] == 1: + eval_contract(block,transactions,tx) + sys.stderr.write("tx processed\n") + def eval(block,transactions,timestamp,coinbase): h = block.hash() # Process all transactions - while len(transactions) > 0: - tx = transactions.pop(0) - fee = 0 - if tx.to = '\x00'*20: - fee += params['newcontractfee'] + len(tx.data) * params['memoryfee'] - else: - fee += params['txfee'] - # Insufficient fee, do nothing - if fee > tx.fee: continue - # Too much data, do nothing - if len(data) > 256: continue - # Try to send the tx - if not block.send(tx): continue - # Evaluate contract if applicable - eval_contract(block,transactions,tx) + process_transactions(block,transactions) # Pay miner fee - miner_state = rlp_decode(self.state.get(self.coinbase)) or ['','',''] - miner_balance = decode(miner_state[1],256) + miner_state = rlp.decode(block.state.get(block.coinbase)) or [0,0,0] block.number += 1 reward = 0 if block.number < params['period_1_duration']: @@ -92,17 +131,17 @@ def eval(block,transactions,timestamp,coinbase): reward = params['period_3_reward'] else: reward = params['period_4_reward'] - miner_balance += reward + print reward + miner_state[1] += reward + block.reward for uncle in block.uncles: - sib_miner_state = rlp_decode(self.state.get(uncle[3])) + sib_miner_state = rlp_decode(block.state.get(uncle[3])) sib_miner_state[1] = encode(decode(sib_miner_state[1],256)+reward*7/8,256) - self.state.update(uncle[3],sib_miner_state) - miner_balance += reward/8 - miner_state[1] = encode(miner_balance,256) - self.state.update(self.coinbase,miner_state) + block.state.update(uncle[3],sib_miner_state) + miner_state[1] += reward/8 + block.state.update(block.coinbase,rlp.encode(miner_state)) # Check timestamp if timestamp < block.timestamp or timestamp > int(time.time()) + 3600: - raise new Exception("timestamp not in valid range!") + raise Exception("timestamp not in valid range!") # Update difficulty if timestamp >= block.timestamp + 42: block.difficulty += int(block.difficulty / 1024) @@ -115,10 +154,11 @@ def eval(block,transactions,timestamp,coinbase): return block def eval_contract(block,transaction_list,tx): + sys.stderr.write("evaluating contract\n") address = tx.to # Initialize registers reg = [0] * 256 - reg[0] = decode(tx.from,256) + reg[0] = decode(tx.sender,256) reg[1] = decode(tx.to,256) reg[2] = tx.value reg[3] = tx.fee @@ -131,8 +171,11 @@ def eval_contract(block,transaction_list,tx): # Convert the data item into a code piece val_at_index = decode(contract.get(encode(index,256,32)),256) code = [ int(val_at_index / (256**i)) % 256 for i in range(6) ] + code[0] = scriptcode_map.get(code[0],'INVALID') + sys.stderr.write("Evaluating: "+ str(code)+"\n") # Invalid code instruction or STOP code stops execution sans fee - if val_at_index >= 256**6 or code[0] == 'STOP': + if val_at_index >= 256**6 or code[0] in ['STOP','INVALID']: + sys.stderr.write("stop code, exiting\n") break # Calculate fee minerfee = 0 @@ -154,10 +197,11 @@ def eval_contract(block,transaction_list,tx): # If we can't pay the fee, break, otherwise pay it if block.get_balance(address) < minerfee + nullfee: + sys.stderr.write("insufficient fee, exiting\n") break - block.pay_fee(address,nullfee,False) - block.pay_fee(address,minerfee,True) - + block.set_balance(address,block.get_balance(address) - nullfee - minerfee) + block.reward += minerfee + sys.stderr.write("evaluating operation\n") # Evaluate operations if c == 'ADD': reg[code[3]] = (reg[code[1]] + reg[code[2]]) % 2**256 @@ -174,7 +218,7 @@ def eval_contract(block,transaction_list,tx): x = reg[code[1]] if reg[code[1]] < 2**255 else 2**256 - reg[code[1]] y = reg[code[2]] if reg[code[2]] < 2**255 else 2**256 - reg[code[2]] z = int(x/y) - reg[code[3]] = z if sign = 1 else 2**256 - z + reg[code[3]] = z if sign == 1 else 2**256 - z elif code == 'MOD': reg[code[3]] = reg[code[1]] % reg[code[2]] elif code == 'SMOD': @@ -184,7 +228,7 @@ def eval_contract(block,transaction_list,tx): x = reg[code[1]] if reg[code[1]] < 2**255 else 2**256 - reg[code[1]] y = reg[code[2]] if reg[code[2]] < 2**255 else 2**256 - reg[code[2]] z = x%y - reg[code[3]] = z if sign = 1 else 2**256 - z + reg[code[3]] = z if sign == 1 else 2**256 - z elif code == 'EXP': reg[code[3]] = pow(reg[code[1]],reg[code[2]],2**256) elif code == 'NEG': @@ -278,7 +322,7 @@ def eval_contract(block,transaction_list,tx): ind = encode((reg[code[5]] + i) % 2**256,256,32) data.append(contract.get(ind)) tx = Transaction(0,to,value,fee,data) - tx.from = address + tx.sender = address transaction_list.insert(0,tx) elif code == 'DATA': reg[code[2]] = tx.data[reg[code[1]]] diff --git a/rlp.py b/rlp.py index 03272b6c7c..0abfffb4ae 100644 --- a/rlp.py +++ b/rlp.py @@ -17,14 +17,14 @@ def from_binary(b): else: return from_binary(b[:-1]) * 256 + ord(b[-1]) def __decode(s,pos=0): - if s == '': + if not s: return (None, 0) else: fchar = ord(s[pos]) - if fchar <= 24: + if fchar < 24: return (ord(s[pos]), pos+1) elif fchar < 56: - b = ord(s[pos]) - 24 + b = ord(s[pos]) - 23 return (from_binary(s[pos+1:pos+1+b]), pos+1+b) elif fchar < 64: b = ord(s[pos]) - 55 @@ -59,11 +59,13 @@ def decode(s): return __decode(s)[0] def encode(s): if isinstance(s,(int,long)): - if s <= 24: + if s < 0: + raise Exception("can't handle negative ints") + elif s >= 0 and s < 24: return chr(s) elif s <= 2**256: b = to_binary(s) - return chr(len(b) + 24) + b + return chr(len(b) + 23) + b else: b = to_binary(s) b2 = to_binary(len(b)) diff --git a/transactions.py b/transactions.py index ef457190b1..c7985bcb99 100644 --- a/transactions.py +++ b/transactions.py @@ -4,6 +4,7 @@ import re class Transaction(): def __init__(*args): + self = args[0] if len(args) == 2: self.parse(args[1]) else: @@ -17,24 +18,27 @@ class Transaction(): if re.match('^[0-9a-fA-F]*$',data): data = data.decode('hex') o = rlp.unparse(data) - self.nonce = decode(o[0],256) + self.nonce = o[0] self.to = o[1] - self.value = decode(o[2],256) - self.fee = decode(o[3],256) - self.data = rlp.unparse(o[4]) + self.value = o[2] + self.fee = o[3] + self.data = rlp.decode(o[4]) self.v = o[5] self.r = o[6] self.s = o[7] rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) - self.from = hash160(ecdsa_raw_recover(rawhash,(self.v,self.r,self.s))) + pub = encode_pubkey(ecdsa_raw_recover(rawhash,(self.v,self.r,self.s)),'bin') + self.sender = bin_sha256(pub[1:])[-20:] + return self def sign(self,key): - rawhash = sha256(rlp.parse([self.to,self.value,self.fee,self.data])) + rawhash = sha256(rlp.encode([self.to,self.value,self.fee,self.data])) self.v,self.r,self.s = ecdsa_raw_sign(rawhash,key) - self.from = hash160(privtopub(key)) + self.sender = bin_sha256(privtopub(key)[1:])[-20:] + return self def serialize(self): - return rlp.parse([self.nonce, self.to, self.value, self.fee, self.data, self.v, self.r, self.s]) + return rlp.encode([self.nonce, self.to, self.value, self.fee, self.data, self.v, self.r, self.s]) def hex_serialize(self): return self.serialize().encode('hex') diff --git a/trie.py b/trie.py index 0b7ff1365d..93702fe2ab 100644 --- a/trie.py +++ b/trie.py @@ -164,6 +164,45 @@ class Trie(): if curnode[16]: total += 1 return total + def __to_dict(self,node): + if not node: return {} + curnode = rlp.decode(self.db.get(node)) + if not curnode: + raise Exception("node not found in database") + if len(curnode) == 2: + lkey = self.__decode_key(curnode[0]) + o = {} + if lkey[-1] == 16: + o[curnode[0]] = curnode[1] + else: + d = self.__to_dict(curnode[1]) + for v in d: + subkey = self.__decode_key(v) + totalkey = self.__encode_key(lkey+subkey) + o[totalkey] = d[v] + return o + elif len(curnode) == 17: + o = {} + for i in range(16): + d = self.__to_dict(curnode[i]) + for v in d: + subkey = self.__decode_key(v) + totalkey = self.__encode_key([i] + subkey) + o[totalkey] = d[v] + if curnode[16]: o[chr(16)] = curnode[16] + return o + else: + raise Exception("bad curnode! "+curnode) + + def to_dict(self,as_hex=False): + d = self.__to_dict(self.root) + o = {} + for v in d: + v2 = ''.join(['0123456789abcdef'[x] for x in self.__decode_key(v)[:-1]]) + if not as_hex: v2 = v2.decode('hex') + o[v2] = d[v] + return o + def get(self,key): key2 = ['0123456789abcdef'.find(x) for x in key.encode('hex')] + [16] return self.__get_state(self.root,key2) From 8e2a08ffde4f72909f1468a679b2530b629203d6 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sun, 29 Dec 2013 20:10:58 -0500 Subject: [PATCH 10/18] Changed python code to new encoding --- trie.py | 66 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/trie.py b/trie.py index 93702fe2ab..18fe25b337 100644 --- a/trie.py +++ b/trie.py @@ -12,6 +12,25 @@ class DB(): def put(self,key,value): return self.db.Put(key,value) def delete(self,key): return self.db.Delete(key) +def hexarraykey_to_bin(key): + term = 1 if key[-1] == 16 else 0 + if term: key2 = key[:-1] + oddlen = len(key) % 2 + flags = 2 * term + oddlen + if oddlen: key = [flags] + key + else: key = [flags,0] + key + o = '' + for i in range(0,len(key),2): + o += chr(16 * key[i] + key[i+1]) + return o + +def bin_to_hexarraykey(bindata): + o = ['0123456789abcdef'.find(x) for x in key[1:].encode('hex')] + if o[0] >= 2: o.append(16) + if o[0] % 2 == 1: o = o[1:] + else: o = o[2:] + return o + databases = {} class Trie(): @@ -21,19 +40,6 @@ class Trie(): if dbfile not in databases: databases[dbfile] = DB(dbfile) self.db = databases[dbfile] - - def __encode_key(self,key): - term = 1 if key[-1] == 16 else 0 - oddlen = (len(key) - term) % 2 - prefix = ('0' if oddlen else '') - main = ''.join(['0123456789abcdef'[x] for x in key[:len(key)-term]]) - return chr(2 * term + oddlen) + (prefix+main).decode('hex') - - def __decode_key(self,key): - o = ['0123456789abcdef'.find(x) for x in key[1:].encode('hex')] - if key[0] == '\x01' or key[0] == '\x03': o = o[1:] - if key[0] == '\x02' or key[0] == '\x03': o.append(16) - return o def __get_state(self,node,key): if self.debug: print 'nk',node.encode('hex'),key @@ -45,7 +51,7 @@ class Trie(): raise Exception("node not found in database") elif len(curnode) == 2: (k2,v2) = curnode - k2 = self.__decode_key(k2) + k2 = hexarraykey_to_bin(k2) if len(key) >= len(k2) and k2 == key[:len(k2)]: return self.__get_state(v2,key[len(k2):]) else: @@ -69,7 +75,7 @@ class Trie(): return value else: if not node: - newnode = [ self.__encode_key(key), value ] + newnode = [ hexarraykey_to_bin(key), value ] return self.__put(newnode) curnode = rlp.decode(self.db.get(node)) if self.debug: print 'icn', curnode @@ -77,9 +83,9 @@ class Trie(): raise Exception("node not found in database") if len(curnode) == 2: (k2, v2) = curnode - k2 = self.__decode_key(k2) + k2 = hexarraykey_to_bin(k2) if key == k2: - newnode = [ self.__encode_key(key), value ] + newnode = [ hexarraykey_to_bin(key), value ] return self.__put(newnode) else: i = 0 @@ -96,7 +102,7 @@ class Trie(): if i == 0: return newhash3 else: - newnode4 = [ self.__encode_key(key[:i]), newhash3 ] + newnode4 = [ hexarraykey_to_bin(key[:i]), newhash3 ] return self.__put(newnode4) else: newnode = [ curnode[i] for i in range(17) ] @@ -114,15 +120,15 @@ class Trie(): if self.debug: print 'dcn', curnode if len(curnode) == 2: (k2, v2) = curnode - k2 = self.__decode_key(k2) + k2 = hexarraykey_to_bin(k2) if key == k2: return '' elif key[:len(k2)] == k2: newhash = self.__delete_state(v2,key[len(k2):]) childnode = rlp.decode(self.db.get(newhash)) if len(childnode) == 2: - newkey = k2 + self.__decode_key(childnode[0]) - newnode = [ self.__encode_key(newkey), childnode[1] ] + newkey = k2 + hexarraykey_to_bin(childnode[0]) + newnode = [ hexarraykey_to_bin(newkey), childnode[1] ] else: newnode = [ curnode[0], newhash ] return self.__put(newnode) @@ -142,8 +148,8 @@ class Trie(): if len(childnode) == 17: newnode2 = [ key[0], newnode[onlynode] ] elif len(childnode) == 2: - newkey = [onlynode] + self.__decode_key(childnode[0]) - newnode2 = [ self.__encode_key(newkey), childnode[1] ] + newkey = [onlynode] + hexarraykey_to_bin(childnode[0]) + newnode2 = [ hexarraykey_to_bin(newkey), childnode[1] ] else: newnode2 = newnode return self.__put(newnode2) @@ -154,7 +160,7 @@ class Trie(): if not curnode: raise Exception("node not found in database") if len(curnode) == 2: - key = self.__decode_key(curnode[0]) + key = hexarraykey_to_bin(curnode[0]) if key[-1] == 16: return 1 else: return self.__get_size(curnode[1]) elif len(curnode) == 17: @@ -170,15 +176,15 @@ class Trie(): if not curnode: raise Exception("node not found in database") if len(curnode) == 2: - lkey = self.__decode_key(curnode[0]) + lkey = hexarraykey_to_bin(curnode[0]) o = {} if lkey[-1] == 16: o[curnode[0]] = curnode[1] else: d = self.__to_dict(curnode[1]) for v in d: - subkey = self.__decode_key(v) - totalkey = self.__encode_key(lkey+subkey) + subkey = hexarraykey_to_bin(v) + totalkey = hexarraykey_to_bin(lkey+subkey) o[totalkey] = d[v] return o elif len(curnode) == 17: @@ -186,8 +192,8 @@ class Trie(): for i in range(16): d = self.__to_dict(curnode[i]) for v in d: - subkey = self.__decode_key(v) - totalkey = self.__encode_key([i] + subkey) + subkey = hexarraykey_to_bin(v) + totalkey = hexarraykey_to_bin([i] + subkey) o[totalkey] = d[v] if curnode[16]: o[chr(16)] = curnode[16] return o @@ -198,7 +204,7 @@ class Trie(): d = self.__to_dict(self.root) o = {} for v in d: - v2 = ''.join(['0123456789abcdef'[x] for x in self.__decode_key(v)[:-1]]) + v2 = ''.join(['0123456789abcdef'[x] for x in hexarraykey_to_bin(v)[:-1]]) if not as_hex: v2 = v2.decode('hex') o[v2] = d[v] return o From 9115adbcca95a457602fc080fe751931c754d66d Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 30 Dec 2013 16:49:41 -0500 Subject: [PATCH 11/18] Switched to stack-based language --- mining.py | 56 ++++++++ processblock.py | 332 ++++++++++++++++++++++++++++-------------------- 2 files changed, 252 insertions(+), 136 deletions(-) create mode 100644 mining.py diff --git a/mining.py b/mining.py new file mode 100644 index 0000000000..be651881d9 --- /dev/null +++ b/mining.py @@ -0,0 +1,56 @@ +import hashlib + +def bin_sha256(x): return hashlib.sha256(x).digest() + +def spread(L): return 16 if L == 9 else 3 + +def nodes(L): return 2**25 if L == 9 else 8**L + +def to_binary(x): return '' if x == 0 else to_binary(int(x / 256)) + chr(x % 256) + +def from_binary(x): return 0 if x == '' else 256 * from_binary(x[:-1]) + ord(x[-1]) + +def mine(root,difficulty,extranonce): + layers = [[] for x in range(9)] + layers[0] = [root] + for L in range(1,10): + prefix = root + to_binary(extranonce) + to_binary(L) + for i in range(nodes(L)): + p = [] + for k in range(spread(L)): + h = bin_sha256(prefix + to_binary(i) + to_binary(k)) + ind = from_binary(h) % nodes(L-1) + p.append(layers[L-1][ind]) + layers[L].append(bin_sha256(''.join(p))) + print "Computed level ",L + prefix = root + to_binary(extranonce) + for i in range(2**26): + p = [] + for k in range(4): + h = bin_sha256(prefix + to_binary(i) + to_binary(k)) + ind = from_binary(h) % nodes(9) + p.append(layers[9][ind]) + h = from_binary(bin_sha256(''.join(p))) + if h * difficulty <= 2**256: + return i + return None + +def verify(root,difficulty,extranonce,nonce): + layers = [{} for x in range(9)] + layers[0] = [root] + def getnode(L,i): + if i not in layers[L]: + p = [] + for k in range(spread(L)): + h = bin_sha256(root + to_binary(extranonce) + to_binary(L) + to_binary(o) + to_binary(k)) + ind = from_binary(h) % nodes(L-1) + p.append(getnode(L-1,ind)) + layers[L][i] = bin_sha256(''.join(p)) + return layers[L][i] + p = [] + for k in range(4): + h = bin_sha256(root + to_binary(extranonce) + to_binary(nonce) + to_binary(k)) + ind = from_binary(h) % nodes(9) + p.append(getnode(9,ind)) + h = from_binary(bin_sha256(''.join(p))) + return h * difficulty <= 2**256 diff --git a/processblock.py b/processblock.py index 64a233ac35..4596162f76 100644 --- a/processblock.py +++ b/processblock.py @@ -3,50 +3,62 @@ from blocks import Block import time import sys import rlp +import math scriptcode_map = { 0x00: 'STOP', - 0x10: 'ADD', - 0x11: 'SUB', - 0x12: 'MUL', - 0x13: 'DIV', - 0x14: 'SDIV', - 0x15: 'MOD', - 0x16: 'SMOD', - 0x17: 'EXP', - 0x18: 'NEG', - 0x20: 'LT', - 0x21: 'LE', - 0x22: 'GT', - 0x23: 'GE', - 0x24: 'EQ', - 0x25: 'NOT', - 0x30: 'SHA256', - 0x31: 'RIPEMD-160', - 0x32: 'ECMUL', - 0x33: 'ECADD', - 0x34: 'ECSIGN', - 0x35: 'ECRECOVER', - 0x40: 'COPY', - 0x41: 'STORE', - 0x42: 'LOAD', - 0x43: 'SET', - 0x50: 'JMP', - 0x51: 'JMPI', - 0x52: 'IND', - 0x60: 'EXTRO', - 0x61: 'BALANCE', - 0x70: 'MKTX', - 0x71: 'RAWTX', - 0x80: 'DATA', - 0x81: 'DATAN', - 0x90: 'MYADDRESS', - 0x91: 'BLKHASH', + 0x01: 'ADD', + 0x02: 'SUB', + 0x03: 'MUL', + 0x04: 'DIV', + 0x05: 'SDIV', + 0x06: 'MOD', + 0x07: 'SMOD', + 0x08: 'EXP', + 0x09: 'NEG', + 0x0a: 'LT', + 0x0b: 'LE', + 0x0c: 'GT', + 0x0d: 'GE', + 0x0e: 'EQ', + 0x0f: 'NOT', + 0x10: 'MYADDRESS', + 0x11: 'TXSENDER', + 0x12: 'TXVALUE', + 0x13: 'TXFEE', + 0x14: 'TXDATAN', + 0x15: 'TXDATA', + 0x16: 'BLK_PREVHASH', + 0x17: 'BLK_COINBASE', + 0x18: 'BLK_TIMESTAMP', + 0x19: 'BLK_NUMBER', + 0x1a: 'BLK_DIFFICULTY', + 0x20: 'SHA256', + 0x21: 'RIPEMD160', + 0x22: 'ECMUL', + 0x23: 'ECADD', + 0x24: 'ECSIGN', + 0x25: 'ECRECOVER', + 0x26: 'ECVALID', + 0x30: 'PUSH', + 0x31: 'POP', + 0x32: 'DUP', + 0x33: 'DUPN', + 0x34: 'SWAP', + 0x35: 'SWAPN', + 0x36: 'LOAD', + 0x37: 'STORE', + 0x40: 'JMP', + 0x41: 'JMPI', + 0x42: 'IND', + 0x50: 'EXTRO', + 0x51: 'BALANCE', + 0x60: 'MKTX', 0xff: 'SUICIDE' } params = { - 'stepfee': 2**64 / 64, + 'stepfee': 10**16, 'txfee': 2**64, 'newcontractfee': 2**64, 'memoryfee': 2**64 / 4, @@ -156,12 +168,8 @@ def eval(block,transactions,timestamp,coinbase): def eval_contract(block,transaction_list,tx): sys.stderr.write("evaluating contract\n") address = tx.to - # Initialize registers - reg = [0] * 256 - reg[0] = decode(tx.sender,256) - reg[1] = decode(tx.to,256) - reg[2] = tx.value - reg[3] = tx.fee + # Initialize stack + stack = [] index = 0 stepcounter = 0 contract = block.get_contract(address) @@ -174,7 +182,7 @@ def eval_contract(block,transaction_list,tx): code[0] = scriptcode_map.get(code[0],'INVALID') sys.stderr.write("Evaluating: "+ str(code)+"\n") # Invalid code instruction or STOP code stops execution sans fee - if val_at_index >= 256**6 or code[0] in ['STOP','INVALID']: + if val_at_index >= 256 or code[0] in ['STOP','INVALID']: sys.stderr.write("stop code, exiting\n") break # Calculate fee @@ -202,141 +210,193 @@ def eval_contract(block,transaction_list,tx): block.set_balance(address,block.get_balance(address) - nullfee - minerfee) block.reward += minerfee sys.stderr.write("evaluating operation\n") + exit = False + def stack_pop(n): + if len(stack) < n: + sys.stderr.write("Stack height insufficient, exiting") + exit = True + return [0] * n + o = stack[-n:] + stack = stack[:-n] + return o # Evaluate operations if c == 'ADD': - reg[code[3]] = (reg[code[1]] + reg[code[2]]) % 2**256 + x,y = stack_pop(2) + stack.append((x + y) % 2**256) elif c == 'MUL': - reg[code[3]] = (reg[code[1]] * reg[code[2]]) % 2**256 + x,y = stack_pop(2) + stack.append((x * y) % 2**256) elif c == 'SUB': - reg[code[3]] = (reg[code[1]] + 2**256 - reg[code[2]]) % 2**256 + x,y = stack_pop(2) + stack.append((x - y) % 2**256) elif c == 'DIV': - reg[code[3]] = int(reg[code[1]] / reg[code[2]]) + x,y = stack_pop(2) + if y == 0: break + stack.append(int(x / y)) elif c == 'SDIV': - sign = 1 - sign *= (1 if reg[code[1]] < 2**255 else -1) - sign *= (1 if reg[code[2]] < 2**255 else -1) - x = reg[code[1]] if reg[code[1]] < 2**255 else 2**256 - reg[code[1]] - y = reg[code[2]] if reg[code[2]] < 2**255 else 2**256 - reg[code[2]] - z = int(x/y) - reg[code[3]] = z if sign == 1 else 2**256 - z + x,y = stack_pop(2) + if y == 0: break + sign = (1 if x < 2**255 else -1) * (1 if y < 2**255 else -1) + xx = x if x < 2**255 else 2**256 - x + yy = y if y < 2**255 else 2**256 - y + z = int(xx/yy) + stack.append(z if sign == 1 else 2**256 - z) elif code == 'MOD': - reg[code[3]] = reg[code[1]] % reg[code[2]] + x,y = stack_pop(2) + if y == 0: break + stack.append(x % y) elif code == 'SMOD': - sign = 1 - sign *= (1 if reg[code[1]] < 2**255 else -1) - sign *= (1 if reg[code[2]] < 2**255 else -1) - x = reg[code[1]] if reg[code[1]] < 2**255 else 2**256 - reg[code[1]] - y = reg[code[2]] if reg[code[2]] < 2**255 else 2**256 - reg[code[2]] - z = x%y - reg[code[3]] = z if sign == 1 else 2**256 - z + x,y = stack_pop(2) + if y == 0: break + sign = (1 if x < 2**255 else -1) * (1 if y < 2**255 else -1) + xx = x if x < 2**255 else 2**256 - x + yy = y if y < 2**255 else 2**256 - y + z = xx%yy + stack.append(z if sign == 1 else 2**256 - z) elif code == 'EXP': - reg[code[3]] = pow(reg[code[1]],reg[code[2]],2**256) + x,y = stack_pop(2) + stack.append(pow(x,y,2**256)) elif code == 'NEG': - reg[code[2]] = 2**256 - reg[code[1]] + stack.append(2**256 - stack.pop(1)[0]) elif code == 'LT': - reg[code[3]] = 1 if reg[code[1]] < reg[code[2]] else 0 + x,y = stack_pop(2) + stack.append(1 if x < y else 0) elif code == 'LE': - reg[code[3]] = 1 if reg[code[1]] <= reg[code[2]] else 0 + x,y = stack_pop(2) + stack.append(1 if x <= y else 0) elif code == 'GT': - reg[code[3]] = 1 if reg[code[1]] > reg[code[2]] else 0 + x,y = stack_pop(2) + stack.append(1 if x > y else 0) elif code == 'GE': - reg[code[3]] = 1 if reg[code[1]] >= reg[code[2]] else 0 + x,y = stack_pop(2) + stack.append(1 if x >= y else 0) elif code == 'EQ': - reg[code[3]] = 1 if reg[code[1]] == reg[code[2]] else 0 + x,y = stack_pop(2) + stack.append(1 if x == y else 0) elif code == 'NOT': - reg[code[2]] = 1 if reg[code[1]] == 0 else 0 + stack.append(1 if stack.pop(1)[0] == 0 else 0) + elif code == 'MYADDRESS': + stack.append(address) + elif code == 'TXSENDER': + stack.append(decode(tx.sender,256)) + elif code == 'TXVALUE': + stack.append(tx.value) + elif code == 'TXFEE': + stack.append(tx.fee) + elif code == 'TXDATAN': + stack.append(len(tx.data)) + elif code == 'TXDATA': + x, = stack_pop(1) + stack.append(0 if x >= len(tx.data) else tx.data[x]) + elif code == 'BLK_PREVHASH': + stack.append(decode(block.prevhash,256)) + elif code == 'BLK_COINBASE': + stack.append(decode(block.coinbase,160)) + elif code == 'BLK_TIMESTAMP': + stack.append(block.timestamp) + elif code == 'BLK_NUMBER': + stack.append(block.number) + elif code == 'BLK_DIFFICULTY': + stack.append(block.difficulty) elif code == 'SHA256': - inp = encode(reg[code[1]],256,32) - reg[code[2]] = decode(hashlib.sha256(inp).digest(),256) + L = stack_pop(1) + hdataitems = stack_pop(math.ceil(L / 32.0)) + hdata = ''.join([encode(x,256,32) for x in hdataitems])[:L] + stack.append(decode(hashlib.sha256(hdata).digest(),256)) elif code == 'RIPEMD-160': - inp = encode(reg[code[1]],256,32) - reg[code[2]] = decode(hashlib.new('ripemd160',inp).digest(),256) + L = stack_pop(1) + hdataitems = stack_pop(math.ceil(L / 32.0)) + hdata = ''.join([encode(x,256,32) for x in hdataitems])[:L] + stack.append(decode(hashlib.new('ripemd160',hdata).digest(),256)) elif code == 'ECMUL': - pt = (reg[code[1]],reg[code[2]]) + n,x,y = stack_pop(3) # Point at infinity - if pt[0] == 0 and pt[1] == 0: - reg[code[4]], reg[code[5]] = 0,0 + if x == 0 and y == 0: + stack.extend([0,0]) # Point not on curve, coerce to infinity - elif (pt[0] ** 3 + 7 - pt[1] ** 2) % N != 0: - reg[code[4]], reg[code[5]] = 0,0 + elif x >= P or y >= P or (x ** 3 + 7 - y ** 2) % P != 0: + stack.extend([0,0]) # Legitimate point else: - pt2 = base10_multiply(pt,reg[code[3]]) - reg[code[4]], reg[code[5]] = pt2[0], pt2[1] + x2,y2 = base10_multiply((x,y),n) + stack.extend([x2,y2]) elif code == 'ECADD': - pt1 = (reg[code[1]],reg[code[2]]) - pt2 = (reg[code[3]],reg[code[4]]) + x1,y1,x2,y2 = stack_pop(4) # Invalid point 1 - if (pt1[0] ** 3 + 7 - pt1[1] ** 2) % N != 0: - reg[code[5]], reg[code[6]] = 0,0 + if x1 >= P or y1 >= P or (x1 ** 3 + 7 - y1 ** 2) % P != 0: + stack.extend([0,0]) # Invalid point 2 - elif (pt2[0] ** 3 + 7 - pt2[1] ** 2) % N != 0: - reg[code[5]], reg[code[6]] = 0,0 + elif x2 >= P or y2 >= P or (x2 ** 3 + 7 - y2 ** 2) % P != 0: + stack.extend([0,0]) # Legitimate points else: - pt3 = base10_add(pt1,pt2) - reg[code[5]], reg[code[6]] = pt3[0], pt3[1] + x3,y3 = base10_add((x1,y1),(x2,y2)) + stack.extend([x3,y3]) elif code == 'ECSIGN': - reg[code[3]], reg[code[4]], reg[code[5]] = ecdsa_raw_sign(reg[code[1]],reg[code[2]]) + k,h = stack_pop(2) + v,r,s = ecdsa_raw_sign(h,k) + stack.extend([v,r,s]) elif code == 'ECRECOVER': - pt = ecdsa_raw_recover((reg[code[2]],reg[code[3]],reg[code[4]]),reg[code[1]]) - reg[code[5]] = pt[0] - reg[code[6]] = pt[1] - elif code == 'COPY': - reg[code[2]] = reg[code[1]] - elif code == 'STORE': - contract.update(encode(reg[code[2]],256,32),reg[code[1]]) + h,v,r,s = stack_pop(4) + x,y = ecdsa_raw_recover((v,r,s),h) + stack.extend([x,y]) + elif code == 'PUSH': + stack.append(contract.get(encode(index + 1,256,32))) + index += 1 + elif code == 'POP': + stack_pop(1) + elif code == 'DUP': + x, = stack_pop(1) + stack.extend([x,x]) + elif code == 'DUPN': + arr = stack_pop(contract.get(encode(index + 1,256,32))) + arr.append(arr[0]) + stack.extend(arr) + index += 1 + elif code == 'SWAP': + x,y = stack_pop(2) + stack.extend([y,x]) + elif code = 'SWAPN': + arr = stack_pop(contract.get(encode(index + 1,256,32))) + arr.append(arr[0]) + arr.pop(0) + stack.extend(arr) + index += 1 elif code == 'LOAD': - reg[code[2]] = contract.get(encode(reg[code[1]],256,32)) - elif code == 'SET': - reg[code[1]] = (code[2] + 256 * code[3] + 65536 * code[4] + 16777216 * code[5]) * 2**code[6] % 2**256 + stack.append(contract.get(encode(stack_pop(1)[0],256,32))) + elif code == 'STORE': + x,y = stack_pop(2) + if exit: break + contract.update(encode(x,256,32),y) elif code == 'JMP': - index = reg[code[1]] + index = stack_pop(1)[0] elif code == 'JMPI': - if reg[code[1]]: index = reg[code[2]] + newpos,c = stack_pop(2) + if c != 0: index = newpos elif code == 'IND': - reg[code[1]] = index + stack.append(index) elif code == 'EXTRO': - if reg[code[1]] >= 2**160: - reg[code[3]] = 0 - else: - address = encode(reg[code[1]],256,20) - field = encode(reg[code[2]]) - reg[code[3]] = block.get_contract(address).get(field) + ind,addr = stack_pop(2) + stack.push(block.get_contract(encode(addr,256,20)).get(encode(ind,256,32))) elif code == 'BALANCE': - if reg[code[1]] >= 2**160: - reg[code[2]] = 0 - else: - address = encode(reg[code[1]],256,20) - reg[code[2]] = block.get_balance(address) + stack.push(block.get_balance(encode(stack_pop(1)[0],256,20))) elif code == 'MKTX': - to = encode(reg[code[1]],256,32) - value = reg[code[2]] - fee = reg[code[3]] - if (value + fee) > block.get_balance(address): - pass + datan,fee,value,to = stack_pop(4) + if exit: + break + elif (value + fee) > block.get_balance(address): + break else: - datan = reg[code[4]] - data = [] - for i in range(datan): - ind = encode((reg[code[5]] + i) % 2**256,256,32) - data.append(contract.get(ind)) - tx = Transaction(0,to,value,fee,data) + data = stack_pop(datan) + tx = Transaction(0,encode(to,256,20),value,fee,data) tx.sender = address transaction_list.insert(0,tx) - elif code == 'DATA': - reg[code[2]] = tx.data[reg[code[1]]] - elif code == 'DATAN': - reg[code[1]] = len(tx.data) - elif code == 'MYADDRESS': - reg[code[1]] = address - elif code == 'BLKHASH': - reg[code[1]] = decode(block.hash()) elif code == 'SUICIDE': sz = contract.get_size() negfee = -sz * params["memoryfee"] - toaddress = encode(reg[code[1]],256,20) - block.pay_fee(roaddress,negfee,False) + toaddress = encode(stack_pop(1)[0],256,20) + block.pay_fee(toaddress,negfee,False) contract.root = '' break + if exit: break block.update_contract(address,contract) From 0cd6ec613c8db1caaa2c78471c104c3be1965416 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 31 Dec 2013 10:32:06 -0500 Subject: [PATCH 12/18] Fixes to trie --- trie.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/trie.py b/trie.py index 18fe25b337..b177acbe1d 100644 --- a/trie.py +++ b/trie.py @@ -14,7 +14,7 @@ class DB(): def hexarraykey_to_bin(key): term = 1 if key[-1] == 16 else 0 - if term: key2 = key[:-1] + if term: key = key[:-1] oddlen = len(key) % 2 flags = 2 * term + oddlen if oddlen: key = [flags] + key @@ -25,7 +25,7 @@ def hexarraykey_to_bin(key): return o def bin_to_hexarraykey(bindata): - o = ['0123456789abcdef'.find(x) for x in key[1:].encode('hex')] + o = ['0123456789abcdef'.find(x) for x in bindata.encode('hex')] if o[0] >= 2: o.append(16) if o[0] % 2 == 1: o = o[1:] else: o = o[2:] @@ -51,7 +51,7 @@ class Trie(): raise Exception("node not found in database") elif len(curnode) == 2: (k2,v2) = curnode - k2 = hexarraykey_to_bin(k2) + k2 = bin_to_hexarraykey(k2) if len(key) >= len(k2) and k2 == key[:len(k2)]: return self.__get_state(v2,key[len(k2):]) else: @@ -83,7 +83,7 @@ class Trie(): raise Exception("node not found in database") if len(curnode) == 2: (k2, v2) = curnode - k2 = hexarraykey_to_bin(k2) + k2 = bin_to_hexarraykey(k2) if key == k2: newnode = [ hexarraykey_to_bin(key), value ] return self.__put(newnode) @@ -176,14 +176,14 @@ class Trie(): if not curnode: raise Exception("node not found in database") if len(curnode) == 2: - lkey = hexarraykey_to_bin(curnode[0]) + lkey = bin_to_hexarraykey(curnode[0]) o = {} if lkey[-1] == 16: o[curnode[0]] = curnode[1] else: d = self.__to_dict(curnode[1]) for v in d: - subkey = hexarraykey_to_bin(v) + subkey = bin_to_hexarraykey(v) totalkey = hexarraykey_to_bin(lkey+subkey) o[totalkey] = d[v] return o @@ -192,7 +192,7 @@ class Trie(): for i in range(16): d = self.__to_dict(curnode[i]) for v in d: - subkey = hexarraykey_to_bin(v) + subkey = bin_to_hexarraykey(v) totalkey = hexarraykey_to_bin([i] + subkey) o[totalkey] = d[v] if curnode[16]: o[chr(16)] = curnode[16] @@ -204,7 +204,7 @@ class Trie(): d = self.__to_dict(self.root) o = {} for v in d: - v2 = ''.join(['0123456789abcdef'[x] for x in hexarraykey_to_bin(v)[:-1]]) + v2 = ''.join(['0123456789abcdef'[x] for x in bin_to_hexarraykey(v)[:-1]]) if not as_hex: v2 = v2.decode('hex') o[v2] = d[v] return o From 2992c04169b958b857c0703acf097a2762424bfc Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 31 Dec 2013 17:29:43 -0500 Subject: [PATCH 13/18] Bugfixes to trie --- trie.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trie.py b/trie.py index b177acbe1d..de2e27b8c3 100644 --- a/trie.py +++ b/trie.py @@ -120,14 +120,14 @@ class Trie(): if self.debug: print 'dcn', curnode if len(curnode) == 2: (k2, v2) = curnode - k2 = hexarraykey_to_bin(k2) + k2 = bin_to_hexarraykey(k2) if key == k2: return '' elif key[:len(k2)] == k2: newhash = self.__delete_state(v2,key[len(k2):]) childnode = rlp.decode(self.db.get(newhash)) if len(childnode) == 2: - newkey = k2 + hexarraykey_to_bin(childnode[0]) + newkey = k2 + bin_to_hexarraykey(childnode[0]) newnode = [ hexarraykey_to_bin(newkey), childnode[1] ] else: newnode = [ curnode[0], newhash ] @@ -148,7 +148,7 @@ class Trie(): if len(childnode) == 17: newnode2 = [ key[0], newnode[onlynode] ] elif len(childnode) == 2: - newkey = [onlynode] + hexarraykey_to_bin(childnode[0]) + newkey = [onlynode] + bin_to_hexarraykey(childnode[0]) newnode2 = [ hexarraykey_to_bin(newkey), childnode[1] ] else: newnode2 = newnode From 9d47cacc47ec8c30da463944711dfa08a29d7174 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 31 Dec 2013 19:02:38 -0500 Subject: [PATCH 14/18] Added testing code for RLP, hexarraykey encoding, tries --- rlp.py | 8 +++--- runtest.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ trie.py | 16 ++++++----- 3 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 runtest.py diff --git a/rlp.py b/rlp.py index 0abfffb4ae..e35f111941 100644 --- a/rlp.py +++ b/rlp.py @@ -63,19 +63,19 @@ def encode(s): raise Exception("can't handle negative ints") elif s >= 0 and s < 24: return chr(s) - elif s <= 2**256: + elif s < 2**256: b = to_binary(s) return chr(len(b) + 23) + b else: b = to_binary(s) b2 = to_binary(len(b)) return chr(len(b2) + 55) + b2 + b - elif isinstance(s,str): + elif isinstance(s,(str,unicode)): if len(s) < 56: - return chr(len(s) + 64) + s + return chr(len(s) + 64) + str(s) else: b2 = to_binary(len(s)) - return chr(len(b2) + 119) + b2 + s + return chr(len(b2) + 119) + b2 + str(s) elif isinstance(s,list): if len(s) < 56: return chr(len(s) + 128) + ''.join([encode(x) for x in s]) diff --git a/runtest.py b/runtest.py new file mode 100644 index 0000000000..5ef92f5d57 --- /dev/null +++ b/runtest.py @@ -0,0 +1,81 @@ +import json, sys, os +import rlp, trie +import random + +testdir = sys.argv[1] if len(sys.argv) >= 2 else '../tests' + +rlpdata = json.loads(open(os.path.join(testdir,'rlptest.txt')).read()) +for x,y in rlpdata: + yprime = rlp.encode(x).encode('hex') + if yprime != y: print "RLPEncode Mismatch: ",x,y,yprime + xprime = rlp.decode(y.decode('hex')) + jx, jxprime = json.dumps(x), json.dumps(xprime) + if jx != jxprime: print "RLPDecode Mismatch: ",jx,jxprime,y + +hexencodedata = json.loads(open(os.path.join(testdir,'hexencodetest.txt')).read()) + +for x,y in hexencodedata: + yprime = trie.hexarraykey_to_bin(x).encode('hex') + if yprime != y: print "HexEncode Mismatch: ",x,y,yprime + xprime = trie.bin_to_hexarraykey(y.decode('hex')) + jx,jxprime = json.dumps(x), json.dumps(xprime) + if jx != jxprime: print "HexDecode Mismatch: ",jx,jxprime,y + +triedata = json.loads(open(os.path.join(testdir,'trietest.txt')).read()) + +for x,y in triedata: + t0 = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) + for k in x: + t0.update(k,x[k]) + if t0.root.encode('hex') != y: + print "Mismatch with adds only" + continue + t = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) + dummies, reals = [], [] + for k in x: + reals.append([k,x[k]]) + dummies.append(k[:random.randrange(len(k)-1)]) + dummies.append(k+random.choice(dummies)) + dummies.append(k[:random.randrange(len(k)-1)]+random.choice(dummies)) + dummies_to_pop = set([]) + i = 0 + ops = [] + mp = {} + success = [True] + def update(k,v): + t.update(k,v) + if v == '' and k in mp: del mp[k] + else: mp[k] = v + ops.append([k,v,t.root.encode('hex')]) + tn = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) + for k in mp: + tn.update(k,mp[k]) + if tn.root != t.root: + print "Mismatch: " + for op in ops: print op + success[0] = False + while i < len(reals): + s = random.randrange(3) + if s == 0: + update(reals[i][0],reals[i][1]) + i += 1 + elif s == 1: + k,v = random.choice(dummies), random.choice(dummies) + update(k,v) + dummies_to_pop.add(k) + elif s == 2: + if len(dummies_to_pop) > 0: + k = random.choice(list(dummies_to_pop)) + update(k,'') + dummies_to_pop.remove(k) + if not success[0]: + break + if not success[0]: + continue + i = len(reals) * 2 + while len(dummies_to_pop) > 0: + k = random.choice(list(dummies_to_pop)) + update(k,'') + dummies_to_pop.remove(k) + if not success[0]: + break diff --git a/trie.py b/trie.py index de2e27b8c3..d914f88d4a 100644 --- a/trie.py +++ b/trie.py @@ -70,7 +70,7 @@ class Trie(): else: return self.__delete_state(node,key) def __insert_state(self,node,key,value): - if self.debug: print 'ink', node.encode('hex'), key + if self.debug: print 'ins', node.encode('hex'), key if len(key) == 0: return value else: @@ -141,12 +141,14 @@ class Trie(): if newnode[i]: if onlynode == -1: onlynode = i else: onlynode = -2 - if onlynode >= 0: + if onlynode == 16: + newnode2 = [ hexarraykey_to_bin([16]), newnode[onlynode] ] + elif onlynode >= 0: childnode = rlp.decode(self.db.get(newnode[onlynode])) if not childnode: raise Exception("?????") if len(childnode) == 17: - newnode2 = [ key[0], newnode[onlynode] ] + newnode2 = [ hexarraykey_to_bin([onlynode]), newnode[onlynode] ] elif len(childnode) == 2: newkey = [onlynode] + bin_to_hexarraykey(childnode[0]) newnode2 = [ hexarraykey_to_bin(newkey), childnode[1] ] @@ -210,13 +212,13 @@ class Trie(): return o def get(self,key): - key2 = ['0123456789abcdef'.find(x) for x in key.encode('hex')] + [16] + key2 = ['0123456789abcdef'.find(x) for x in str(key).encode('hex')] + [16] return self.__get_state(self.root,key2) def get_size(self): return self.__get_size(self.root) def update(self,key,value): - if not isinstance(key,str) or not isinstance(value,str): + if not isinstance(key,(str,unicode)) or not isinstance(value,(str,unicode)): raise Exception("Key and value must be strings") - key2 = ['0123456789abcdef'.find(x) for x in key.encode('hex')] + [16] - self.root = self.__update_state(self.root,key2,value) + key2 = ['0123456789abcdef'.find(x) for x in str(key).encode('hex')] + [16] + self.root = self.__update_state(self.root,key2,str(value)) From f1208195f4ebe58d270aefe489ae3bdd7208873e Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 7 Jan 2014 12:40:49 -0500 Subject: [PATCH 15/18] Updated trie format and stack language --- processblock.py | 49 ++++++++++++++++++++++++++----------------------- transactions.py | 6 +++--- trie.py | 29 ++++++++++++++++++----------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/processblock.py b/processblock.py index 4596162f76..a681a5970c 100644 --- a/processblock.py +++ b/processblock.py @@ -58,23 +58,27 @@ scriptcode_map = { } params = { - 'stepfee': 10**16, - 'txfee': 2**64, - 'newcontractfee': 2**64, - 'memoryfee': 2**64 / 4, - 'datafee': 2**64 / 16, - 'cryptofee': 2**64 / 16, - 'extrofee': 2**64 / 16, + 'stepfee': 1, + 'txfee': 100, + 'newcontractfee': 100, + 'memoryfee': 20, + 'datafee': 4, + 'cryptofee': 10, + 'extrofee': 10, 'blocktime': 60, - 'period_1_reward': 2**80 * 1024, + 'period_1_reward': 10**18 * 800, 'period_1_duration': 57600, - 'period_2_reward': 2**80 * 512, + 'period_2_reward': 10**18 * 400, 'period_2_duration': 57600, - 'period_3_reward': 2**80 * 256, + 'period_3_reward': 10**18 * 100, 'period_3_duration': 57600, - 'period_4_reward': 2**80 * 128 + 'period_4_reward': 10**18 * 50 } +def getfee(block,t): + if t in ['stepfee','txfee','newcontractfee','memoryfee','datafee','cryptofee','extrofee']: + return int(10**24 / int(block.difficulty ** 0.5)) * params[t] + def process_transactions(block,transactions): while len(transactions) > 0: tx = transactions.pop(0) @@ -85,9 +89,9 @@ def process_transactions(block,transactions): tdata = rlp.decode(block.state.get(tx.to)) or [0,0,0] # Calculate fee if tx.to == '\x00'*20: - fee = params['newcontractfee'] + len(tx.data) * params['memoryfee'] + fee = getfee('newcontractfee') + len(tx.data) * getfee('memoryfee') else: - fee = params['txfee'] + fee = getfee('txfee') # Insufficient fee, do nothing if fee > tx.fee: sys.stderr.write("Insufficient fee\n") @@ -143,11 +147,10 @@ def eval(block,transactions,timestamp,coinbase): reward = params['period_3_reward'] else: reward = params['period_4_reward'] - print reward miner_state[1] += reward + block.reward for uncle in block.uncles: sib_miner_state = rlp_decode(block.state.get(uncle[3])) - sib_miner_state[1] = encode(decode(sib_miner_state[1],256)+reward*7/8,256) + sib_miner_state[1] += reward*7/8 block.state.update(uncle[3],sib_miner_state) miner_state[1] += reward/8 block.state.update(block.coinbase,rlp.encode(miner_state)) @@ -190,18 +193,18 @@ def eval_contract(block,transaction_list,tx): nullfee = 0 stepcounter += 1 if stepcounter > 16: - minerfee += params["stepfee"] + minerfee += getfee("stepfee") c = scriptcode_map[code[0]] if c in ['STORE','LOAD']: - minerfee += params["datafee"] + minerfee += getfee("datafee") if c in ['EXTRO','BALANCE']: - minerfee += params["extrofee"] + minerfee += getfee("extrofee") if c in ['SHA256','RIPEMD-160','ECMUL','ECADD','ECSIGN','ECRECOVER']: - minerfee += params["cryptofee"] + minerfee += getfee("cryptofee") if c == 'STORE': existing = block.get_contract_state(address,code[2]) - if reg[code[1]] != 0: nullfee += params["memoryfee"] - if existing: nullfee -= params["memoryfee"] + if reg[code[1]] != 0: nullfee += getfee("memoryfee") + if existing: nullfee -= getfee("memoryfee") # If we can't pay the fee, break, otherwise pay it if block.get_balance(address) < minerfee + nullfee: @@ -356,7 +359,7 @@ def eval_contract(block,transaction_list,tx): elif code == 'SWAP': x,y = stack_pop(2) stack.extend([y,x]) - elif code = 'SWAPN': + elif code == 'SWAPN': arr = stack_pop(contract.get(encode(index + 1,256,32))) arr.append(arr[0]) arr.pop(0) @@ -393,7 +396,7 @@ def eval_contract(block,transaction_list,tx): transaction_list.insert(0,tx) elif code == 'SUICIDE': sz = contract.get_size() - negfee = -sz * params["memoryfee"] + negfee = -sz * getfee("memoryfee") toaddress = encode(stack_pop(1)[0],256,20) block.pay_fee(toaddress,negfee,False) contract.root = '' diff --git a/transactions.py b/transactions.py index c7985bcb99..1cd1d74512 100644 --- a/transactions.py +++ b/transactions.py @@ -17,12 +17,12 @@ class Transaction(): def parse(self,data): if re.match('^[0-9a-fA-F]*$',data): data = data.decode('hex') - o = rlp.unparse(data) + o = rlp.decode(data) self.nonce = o[0] self.to = o[1] self.value = o[2] self.fee = o[3] - self.data = rlp.decode(o[4]) + self.data = o[4] self.v = o[5] self.r = o[6] self.s = o[7] @@ -32,7 +32,7 @@ class Transaction(): return self def sign(self,key): - rawhash = sha256(rlp.encode([self.to,self.value,self.fee,self.data])) + rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) self.v,self.r,self.s = ecdsa_raw_sign(rawhash,key) self.sender = bin_sha256(privtopub(key)[1:])[-20:] return self diff --git a/trie.py b/trie.py index d914f88d4a..c5cc7e563a 100644 --- a/trie.py +++ b/trie.py @@ -1,8 +1,8 @@ import leveldb import rlp -import hashlib +from sha3 import sha3_256 -def sha256(x): return hashlib.sha256(x).digest() +def sha3(x): return sha3_256(x).digest() class DB(): def __init__(self,dbfile): self.db = leveldb.LevelDB(dbfile) @@ -45,7 +45,7 @@ class Trie(): if self.debug: print 'nk',node.encode('hex'),key if len(key) == 0 or not node: return node - curnode = rlp.decode(self.db.get(node)) + curnode = rlp.decode(self.__lookup(node)) if self.debug: print 'cn', curnode if not curnode: raise Exception("node not found in database") @@ -61,10 +61,17 @@ class Trie(): def __put(self,node): rlpnode = rlp.encode(node) - h = sha256(rlpnode) - self.db.put(h,rlpnode) + if len(rlpnode) >= 32: + h = sha3(rlpnode) + self.db.put(h,rlpnode) + else: + h = rlpnode return h + def __lookup(self,node): + if len(node) < 32: return node + else: return self.db.get(node) + def __update_state(self,node,key,value): if value != '': return self.__insert_state(node,key,value) else: return self.__delete_state(node,key) @@ -77,7 +84,7 @@ class Trie(): if not node: newnode = [ hexarraykey_to_bin(key), value ] return self.__put(newnode) - curnode = rlp.decode(self.db.get(node)) + curnode = rlp.decode(self.__lookup(node)) if self.debug: print 'icn', curnode if not curnode: raise Exception("node not found in database") @@ -114,7 +121,7 @@ class Trie(): if len(key) == 0 or not node: return '' else: - curnode = rlp.decode(self.db.get(node)) + curnode = rlp.decode(self.__lookup(node)) if not curnode: raise Exception("node not found in database") if self.debug: print 'dcn', curnode @@ -125,7 +132,7 @@ class Trie(): return '' elif key[:len(k2)] == k2: newhash = self.__delete_state(v2,key[len(k2):]) - childnode = rlp.decode(self.db.get(newhash)) + childnode = rlp.decode(self.__lookup(newhash)) if len(childnode) == 2: newkey = k2 + bin_to_hexarraykey(childnode[0]) newnode = [ hexarraykey_to_bin(newkey), childnode[1] ] @@ -144,7 +151,7 @@ class Trie(): if onlynode == 16: newnode2 = [ hexarraykey_to_bin([16]), newnode[onlynode] ] elif onlynode >= 0: - childnode = rlp.decode(self.db.get(newnode[onlynode])) + childnode = rlp.decode(self.__lookup(newnode[onlynode])) if not childnode: raise Exception("?????") if len(childnode) == 17: @@ -158,7 +165,7 @@ class Trie(): def __get_size(self,node): if not node: return 0 - curnode = self.db.get(node) + curnode = self.__lookup(node) if not curnode: raise Exception("node not found in database") if len(curnode) == 2: @@ -174,7 +181,7 @@ class Trie(): def __to_dict(self,node): if not node: return {} - curnode = rlp.decode(self.db.get(node)) + curnode = rlp.decode(self.__lookup(node)) if not curnode: raise Exception("node not found in database") if len(curnode) == 2: From 9c2470902b3a0a1b7669c2cf7996bc9c3dc0daa3 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Wed, 8 Jan 2014 11:26:19 -0500 Subject: [PATCH 16/18] Further update to trie --- trie.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/trie.py b/trie.py index c5cc7e563a..2aa9a3d0fd 100644 --- a/trie.py +++ b/trie.py @@ -45,7 +45,7 @@ class Trie(): if self.debug: print 'nk',node.encode('hex'),key if len(key) == 0 or not node: return node - curnode = rlp.decode(self.__lookup(node)) + curnode = self.lookup(node) if self.debug: print 'cn', curnode if not curnode: raise Exception("node not found in database") @@ -59,18 +59,20 @@ class Trie(): elif len(curnode) == 17: return self.__get_state(curnode[key[0]],key[1:]) - def __put(self,node): + def __put(self,node,root=False): rlpnode = rlp.encode(node) if len(rlpnode) >= 32: h = sha3(rlpnode) self.db.put(h,rlpnode) else: - h = rlpnode + h = rlpnode if root else node return h - def __lookup(self,node): - if len(node) < 32: return node - else: return self.db.get(node) + def lookup(self,node): + if not isinstance(node,(str,unicode)): return node + elif len(node) == 0: return node + elif len(node) < 32: return rlp.decode(node) + else: return rlp.decode(self.db.get(node)) def __update_state(self,node,key,value): if value != '': return self.__insert_state(node,key,value) @@ -84,7 +86,7 @@ class Trie(): if not node: newnode = [ hexarraykey_to_bin(key), value ] return self.__put(newnode) - curnode = rlp.decode(self.__lookup(node)) + curnode = self.lookup(node) if self.debug: print 'icn', curnode if not curnode: raise Exception("node not found in database") @@ -121,7 +123,7 @@ class Trie(): if len(key) == 0 or not node: return '' else: - curnode = rlp.decode(self.__lookup(node)) + curnode = self.lookup(node) if not curnode: raise Exception("node not found in database") if self.debug: print 'dcn', curnode @@ -132,7 +134,7 @@ class Trie(): return '' elif key[:len(k2)] == k2: newhash = self.__delete_state(v2,key[len(k2):]) - childnode = rlp.decode(self.__lookup(newhash)) + childnode = self.lookup(newhash) if len(childnode) == 2: newkey = k2 + bin_to_hexarraykey(childnode[0]) newnode = [ hexarraykey_to_bin(newkey), childnode[1] ] @@ -151,7 +153,7 @@ class Trie(): if onlynode == 16: newnode2 = [ hexarraykey_to_bin([16]), newnode[onlynode] ] elif onlynode >= 0: - childnode = rlp.decode(self.__lookup(newnode[onlynode])) + childnode = self.lookup(newnode[onlynode]) if not childnode: raise Exception("?????") if len(childnode) == 17: @@ -165,7 +167,7 @@ class Trie(): def __get_size(self,node): if not node: return 0 - curnode = self.__lookup(node) + curnode = self.lookup(node) if not curnode: raise Exception("node not found in database") if len(curnode) == 2: @@ -181,7 +183,7 @@ class Trie(): def __to_dict(self,node): if not node: return {} - curnode = rlp.decode(self.__lookup(node)) + curnode = self.lookup(node) if not curnode: raise Exception("node not found in database") if len(curnode) == 2: From 66ba43ed4a94d07c6805e0d29b5a393fbe5b7b36 Mon Sep 17 00:00:00 2001 From: Stephan Tual Date: Wed, 15 Jan 2014 16:05:22 +0000 Subject: [PATCH 17/18] =?UTF-8?q?Corrected=20typo=20=E2=80=98enroding=20>?= =?UTF-8?q?=20encoding=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dev_console.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_console.go b/dev_console.go index 228cfc47e5..5340a5f462 100644 --- a/dev_console.go +++ b/dev_console.go @@ -116,7 +116,7 @@ func (i *Console) ParseInput(input string) bool { "rawroot - Prints the raw merkle root\n" + "\033[1m= Dagger =\033[0m\n" + "dag HASH NONCE - Verifies a nonce with the given hash with dagger\n" + - "\033[1m= Enroding =\033[0m\n" + + "\033[1m= Encoding =\033[0m\n" + "decode STR\n" + "encode STR\n") From bc2664181f1ac51422b5e6d430c6fc81f109c0c5 Mon Sep 17 00:00:00 2001 From: Stephan Tual Date: Wed, 15 Jan 2014 17:26:19 +0000 Subject: [PATCH 18/18] no message --- blocks.py | 100 ------------ manager.py | 89 ----------- mining.py | 56 ------- parser.py | 5 - processblock.py | 405 ------------------------------------------------ rlp.py | 86 ---------- runtest.py | 81 ---------- transactions.py | 47 ------ trie.py | 233 ---------------------------- trietest.py | 25 --- 10 files changed, 1127 deletions(-) delete mode 100644 blocks.py delete mode 100644 manager.py delete mode 100644 mining.py delete mode 100644 parser.py delete mode 100644 processblock.py delete mode 100644 rlp.py delete mode 100644 runtest.py delete mode 100644 transactions.py delete mode 100644 trie.py delete mode 100644 trietest.py diff --git a/blocks.py b/blocks.py deleted file mode 100644 index c9e94c2098..0000000000 --- a/blocks.py +++ /dev/null @@ -1,100 +0,0 @@ -from pybitcointools import * -import rlp -import re -from transactions import Transaction -from trie import Trie -import sys - -class Block(): - def __init__(self,data=None): - - if not data: - return - - if re.match('^[0-9a-fA-F]*$',data): - data = data.decode('hex') - - header, transaction_list, self.uncles = rlp.decode(data) - [ self.number, - self.prevhash, - self.uncles_root, - self.coinbase, - state_root, - self.transactions_root, - self.difficulty, - self.timestamp, - self.nonce, - self.extra ] = header - self.transactions = [Transaction(x) for x in transaction_list] - self.state = Trie('statedb',state_root) - self.reward = 0 - - # Verifications - if self.state.root != '' and self.state.db.get(self.state.root) == '': - raise Exception("State Merkle root not found in database!") - if bin_sha256(rlp.encode(transaction_list)) != self.transactions_root: - raise Exception("Transaction list root hash does not match!") - if bin_sha256(rlp.encode(self.uncles)) != self.uncles_root: - raise Exception("Uncle root hash does not match!") - # TODO: check POW - - def pay_fee(self,address,fee,tominer=True): - # Subtract fee from sender - sender_state = rlp.decode(self.state.get(address)) - if not sender_state or sender_state[1] < fee: - return False - sender_state[1] -= fee - self.state.update(address,sender_state) - # Pay fee to miner - if tominer: - miner_state = rlp.decode(self.state.get(self.coinbase)) or [0,0,0] - miner_state[1] += fee - self.state.update(self.coinbase,miner_state) - return True - - def get_nonce(self,address): - state = rlp.decode(self.state.get(address)) - if not state or state[0] == 0: return False - return state[2] - - def get_balance(self,address): - state = rlp.decode(self.state.get(address)) - return state[1] if state else 0 - - def set_balance(self,address,balance): - state = rlp.decode(self.state.get(address)) or [0,0,0] - state[1] = balance - self.state.update(address,rlp.encode(state)) - - - # Making updates to the object obtained from this method will do nothing. You need - # to call update_contract to finalize the changes. - def get_contract(self,address): - state = rlp.decode(self.state.get(address)) - if not state or state[0] == 0: return False - return Trie('statedb',state[2]) - - def update_contract(self,address,contract): - state = rlp.decode(self.state.get(address)) or [1,0,''] - if state[0] == 0: return False - state[2] = contract.root - self.state.update(address,state) - - # Serialization method; should act as perfect inverse function of the constructor - # assuming no verification failures - def serialize(self): - txlist = [x.serialize() for x in self.transactions] - header = [ self.number, - self.prevhash, - bin_sha256(rlp.encode(self.uncles)), - self.coinbase, - self.state.root, - bin_sha256(rlp.encode(txlist)), - self.difficulty, - self.timestamp, - self.nonce, - self.extra ] - return rlp.encode([header, txlist, self.uncles ]) - - def hash(self): - return bin_sha256(self.serialize()) diff --git a/manager.py b/manager.py deleted file mode 100644 index dbe0f37cdd..0000000000 --- a/manager.py +++ /dev/null @@ -1,89 +0,0 @@ -import rlp -import leveldb -from blocks import Block -from transactions import Transaction -import processblock -import hashlib -from pybitcointools import * - -txpool = {} - -genesis_header = [ - 0, - '', - bin_sha256(rlp.encode([])), - '', - '', - bin_sha256(rlp.encode([])), - 2**36, - 0, - 0, - '' -] - -genesis = [ genesis_header, [], [] ] - -mainblk = Block(rlp.encode(genesis)) - -db = leveldb.LevelDB("objects") - -def genaddr(seed): - priv = bin_sha256(seed) - addr = bin_sha256(privtopub(priv)[1:])[-20:] - return priv,addr - -# For testing -k1,a1 = genaddr("123") -k2,a2 = genaddr("456") - -def broadcast(obj): - pass - -def receive(obj): - d = rlp.decode(obj) - # Is transaction - if len(d) == 8: - tx = Transaction(obj) - if mainblk.get_balance(tx.sender) < tx.value + tx.fee: return - if mainblk.get_nonce(tx.sender) != tx.nonce: return - txpool[bin_sha256(blk)] = blk - broadcast(blk) - # Is message - elif len(d) == 2: - if d[0] == 'getobj': - try: return db.Get(d[1][0]) - except: - try: return mainblk.state.db.get(d[1][0]) - except: return None - elif d[0] == 'getbalance': - try: return mainblk.state.get_balance(d[1][0]) - except: return None - elif d[0] == 'getcontractroot': - try: return mainblk.state.get_contract(d[1][0]).root - except: return None - elif d[0] == 'getcontractsize': - try: return mainblk.state.get_contract(d[1][0]).get_size() - except: return None - elif d[0] == 'getcontractstate': - try: return mainblk.state.get_contract(d[1][0]).get(d[1][1]) - except: return None - # Is block - elif len(d) == 3: - blk = Block(obj) - p = block.prevhash - try: - parent = Block(db.Get(p)) - except: - return - uncles = block.uncles - for s in uncles: - try: - sib = db.Get(s) - except: - return - processblock.eval(parent,blk.transactions,blk.timestamp,blk.coinbase) - if parent.state.root != blk.state.root: return - if parent.difficulty != blk.difficulty: return - if parent.number != blk.number: return - db.Put(blk.hash(),blk.serialize()) - diff --git a/mining.py b/mining.py deleted file mode 100644 index be651881d9..0000000000 --- a/mining.py +++ /dev/null @@ -1,56 +0,0 @@ -import hashlib - -def bin_sha256(x): return hashlib.sha256(x).digest() - -def spread(L): return 16 if L == 9 else 3 - -def nodes(L): return 2**25 if L == 9 else 8**L - -def to_binary(x): return '' if x == 0 else to_binary(int(x / 256)) + chr(x % 256) - -def from_binary(x): return 0 if x == '' else 256 * from_binary(x[:-1]) + ord(x[-1]) - -def mine(root,difficulty,extranonce): - layers = [[] for x in range(9)] - layers[0] = [root] - for L in range(1,10): - prefix = root + to_binary(extranonce) + to_binary(L) - for i in range(nodes(L)): - p = [] - for k in range(spread(L)): - h = bin_sha256(prefix + to_binary(i) + to_binary(k)) - ind = from_binary(h) % nodes(L-1) - p.append(layers[L-1][ind]) - layers[L].append(bin_sha256(''.join(p))) - print "Computed level ",L - prefix = root + to_binary(extranonce) - for i in range(2**26): - p = [] - for k in range(4): - h = bin_sha256(prefix + to_binary(i) + to_binary(k)) - ind = from_binary(h) % nodes(9) - p.append(layers[9][ind]) - h = from_binary(bin_sha256(''.join(p))) - if h * difficulty <= 2**256: - return i - return None - -def verify(root,difficulty,extranonce,nonce): - layers = [{} for x in range(9)] - layers[0] = [root] - def getnode(L,i): - if i not in layers[L]: - p = [] - for k in range(spread(L)): - h = bin_sha256(root + to_binary(extranonce) + to_binary(L) + to_binary(o) + to_binary(k)) - ind = from_binary(h) % nodes(L-1) - p.append(getnode(L-1,ind)) - layers[L][i] = bin_sha256(''.join(p)) - return layers[L][i] - p = [] - for k in range(4): - h = bin_sha256(root + to_binary(extranonce) + to_binary(nonce) + to_binary(k)) - ind = from_binary(h) % nodes(9) - p.append(getnode(9,ind)) - h = from_binary(bin_sha256(''.join(p))) - return h * difficulty <= 2**256 diff --git a/parser.py b/parser.py deleted file mode 100644 index 00823aeaa3..0000000000 --- a/parser.py +++ /dev/null @@ -1,5 +0,0 @@ -import rlp - -def parse(inp): - if inp[0] == '\x00': - return { "type": "transaction", "data": rlp.parse( diff --git a/processblock.py b/processblock.py deleted file mode 100644 index a681a5970c..0000000000 --- a/processblock.py +++ /dev/null @@ -1,405 +0,0 @@ -from transactions import Transaction -from blocks import Block -import time -import sys -import rlp -import math - -scriptcode_map = { - 0x00: 'STOP', - 0x01: 'ADD', - 0x02: 'SUB', - 0x03: 'MUL', - 0x04: 'DIV', - 0x05: 'SDIV', - 0x06: 'MOD', - 0x07: 'SMOD', - 0x08: 'EXP', - 0x09: 'NEG', - 0x0a: 'LT', - 0x0b: 'LE', - 0x0c: 'GT', - 0x0d: 'GE', - 0x0e: 'EQ', - 0x0f: 'NOT', - 0x10: 'MYADDRESS', - 0x11: 'TXSENDER', - 0x12: 'TXVALUE', - 0x13: 'TXFEE', - 0x14: 'TXDATAN', - 0x15: 'TXDATA', - 0x16: 'BLK_PREVHASH', - 0x17: 'BLK_COINBASE', - 0x18: 'BLK_TIMESTAMP', - 0x19: 'BLK_NUMBER', - 0x1a: 'BLK_DIFFICULTY', - 0x20: 'SHA256', - 0x21: 'RIPEMD160', - 0x22: 'ECMUL', - 0x23: 'ECADD', - 0x24: 'ECSIGN', - 0x25: 'ECRECOVER', - 0x26: 'ECVALID', - 0x30: 'PUSH', - 0x31: 'POP', - 0x32: 'DUP', - 0x33: 'DUPN', - 0x34: 'SWAP', - 0x35: 'SWAPN', - 0x36: 'LOAD', - 0x37: 'STORE', - 0x40: 'JMP', - 0x41: 'JMPI', - 0x42: 'IND', - 0x50: 'EXTRO', - 0x51: 'BALANCE', - 0x60: 'MKTX', - 0xff: 'SUICIDE' -} - -params = { - 'stepfee': 1, - 'txfee': 100, - 'newcontractfee': 100, - 'memoryfee': 20, - 'datafee': 4, - 'cryptofee': 10, - 'extrofee': 10, - 'blocktime': 60, - 'period_1_reward': 10**18 * 800, - 'period_1_duration': 57600, - 'period_2_reward': 10**18 * 400, - 'period_2_duration': 57600, - 'period_3_reward': 10**18 * 100, - 'period_3_duration': 57600, - 'period_4_reward': 10**18 * 50 -} - -def getfee(block,t): - if t in ['stepfee','txfee','newcontractfee','memoryfee','datafee','cryptofee','extrofee']: - return int(10**24 / int(block.difficulty ** 0.5)) * params[t] - -def process_transactions(block,transactions): - while len(transactions) > 0: - tx = transactions.pop(0) - enc = (tx.value, tx.fee, tx.sender.encode('hex'), tx.to.encode('hex')) - sys.stderr.write("Attempting to send %d plus fee %d from %s to %s\n" % enc) - # Grab data about sender, recipient and miner - sdata = rlp.decode(block.state.get(tx.sender)) or [0,0,0] - tdata = rlp.decode(block.state.get(tx.to)) or [0,0,0] - # Calculate fee - if tx.to == '\x00'*20: - fee = getfee('newcontractfee') + len(tx.data) * getfee('memoryfee') - else: - fee = getfee('txfee') - # Insufficient fee, do nothing - if fee > tx.fee: - sys.stderr.write("Insufficient fee\n") - continue - # Too much data, do nothing - if len(tx.data) > 256: - sys.stderr.write("Too many data items\n") - continue - if not sdata or sdata[1] < tx.value + tx.fee: - sys.stderr.write("Insufficient funds to send fee\n") - continue - elif tx.nonce != sdata[2] and sdata[0] == 0: - sys.stderr.write("Bad nonce\n") - continue - # Try to send the tx - if sdata[0] == 0: sdata[2] += 1 - sdata[1] -= (tx.value + tx.fee) - block.reward += tx.fee - if tx.to != '': - tdata[1] += tx.value - else: - addr = tx.hash()[-20:] - adata = rlp.decode(block.state.get(addr)) - if adata[2] != '': - sys.stderr.write("Contract already exists\n") - continue - block.state.update(addr,rlp.encode([1,tx.value,''])) - contract = block.get_contract(addr) - for i in range(len(tx.data)): - contract.update(encode(i,256,32),tx.data[i]) - block.update_contract(addr) - print sdata, tdata - block.state.update(tx.sender,rlp.encode(sdata)) - block.state.update(tx.to,rlp.encode(tdata)) - # Evaluate contract if applicable - if tdata[0] == 1: - eval_contract(block,transactions,tx) - sys.stderr.write("tx processed\n") - -def eval(block,transactions,timestamp,coinbase): - h = block.hash() - # Process all transactions - process_transactions(block,transactions) - # Pay miner fee - miner_state = rlp.decode(block.state.get(block.coinbase)) or [0,0,0] - block.number += 1 - reward = 0 - if block.number < params['period_1_duration']: - reward = params['period_1_reward'] - elif block.number < params['period_2_duration']: - reward = params['period_2_reward'] - elif block.number < params['period_3_duration']: - reward = params['period_3_reward'] - else: - reward = params['period_4_reward'] - miner_state[1] += reward + block.reward - for uncle in block.uncles: - sib_miner_state = rlp_decode(block.state.get(uncle[3])) - sib_miner_state[1] += reward*7/8 - block.state.update(uncle[3],sib_miner_state) - miner_state[1] += reward/8 - block.state.update(block.coinbase,rlp.encode(miner_state)) - # Check timestamp - if timestamp < block.timestamp or timestamp > int(time.time()) + 3600: - raise Exception("timestamp not in valid range!") - # Update difficulty - if timestamp >= block.timestamp + 42: - block.difficulty += int(block.difficulty / 1024) - else: - block.difficulty -= int(block.difficulty / 1024) - block.prevhash = h - block.coinbase = coinbase - block.transactions = [] - block.uncles = [] - return block - -def eval_contract(block,transaction_list,tx): - sys.stderr.write("evaluating contract\n") - address = tx.to - # Initialize stack - stack = [] - index = 0 - stepcounter = 0 - contract = block.get_contract(address) - if not contract: - return - while 1: - # Convert the data item into a code piece - val_at_index = decode(contract.get(encode(index,256,32)),256) - code = [ int(val_at_index / (256**i)) % 256 for i in range(6) ] - code[0] = scriptcode_map.get(code[0],'INVALID') - sys.stderr.write("Evaluating: "+ str(code)+"\n") - # Invalid code instruction or STOP code stops execution sans fee - if val_at_index >= 256 or code[0] in ['STOP','INVALID']: - sys.stderr.write("stop code, exiting\n") - break - # Calculate fee - minerfee = 0 - nullfee = 0 - stepcounter += 1 - if stepcounter > 16: - minerfee += getfee("stepfee") - c = scriptcode_map[code[0]] - if c in ['STORE','LOAD']: - minerfee += getfee("datafee") - if c in ['EXTRO','BALANCE']: - minerfee += getfee("extrofee") - if c in ['SHA256','RIPEMD-160','ECMUL','ECADD','ECSIGN','ECRECOVER']: - minerfee += getfee("cryptofee") - if c == 'STORE': - existing = block.get_contract_state(address,code[2]) - if reg[code[1]] != 0: nullfee += getfee("memoryfee") - if existing: nullfee -= getfee("memoryfee") - - # If we can't pay the fee, break, otherwise pay it - if block.get_balance(address) < minerfee + nullfee: - sys.stderr.write("insufficient fee, exiting\n") - break - block.set_balance(address,block.get_balance(address) - nullfee - minerfee) - block.reward += minerfee - sys.stderr.write("evaluating operation\n") - exit = False - def stack_pop(n): - if len(stack) < n: - sys.stderr.write("Stack height insufficient, exiting") - exit = True - return [0] * n - o = stack[-n:] - stack = stack[:-n] - return o - # Evaluate operations - if c == 'ADD': - x,y = stack_pop(2) - stack.append((x + y) % 2**256) - elif c == 'MUL': - x,y = stack_pop(2) - stack.append((x * y) % 2**256) - elif c == 'SUB': - x,y = stack_pop(2) - stack.append((x - y) % 2**256) - elif c == 'DIV': - x,y = stack_pop(2) - if y == 0: break - stack.append(int(x / y)) - elif c == 'SDIV': - x,y = stack_pop(2) - if y == 0: break - sign = (1 if x < 2**255 else -1) * (1 if y < 2**255 else -1) - xx = x if x < 2**255 else 2**256 - x - yy = y if y < 2**255 else 2**256 - y - z = int(xx/yy) - stack.append(z if sign == 1 else 2**256 - z) - elif code == 'MOD': - x,y = stack_pop(2) - if y == 0: break - stack.append(x % y) - elif code == 'SMOD': - x,y = stack_pop(2) - if y == 0: break - sign = (1 if x < 2**255 else -1) * (1 if y < 2**255 else -1) - xx = x if x < 2**255 else 2**256 - x - yy = y if y < 2**255 else 2**256 - y - z = xx%yy - stack.append(z if sign == 1 else 2**256 - z) - elif code == 'EXP': - x,y = stack_pop(2) - stack.append(pow(x,y,2**256)) - elif code == 'NEG': - stack.append(2**256 - stack.pop(1)[0]) - elif code == 'LT': - x,y = stack_pop(2) - stack.append(1 if x < y else 0) - elif code == 'LE': - x,y = stack_pop(2) - stack.append(1 if x <= y else 0) - elif code == 'GT': - x,y = stack_pop(2) - stack.append(1 if x > y else 0) - elif code == 'GE': - x,y = stack_pop(2) - stack.append(1 if x >= y else 0) - elif code == 'EQ': - x,y = stack_pop(2) - stack.append(1 if x == y else 0) - elif code == 'NOT': - stack.append(1 if stack.pop(1)[0] == 0 else 0) - elif code == 'MYADDRESS': - stack.append(address) - elif code == 'TXSENDER': - stack.append(decode(tx.sender,256)) - elif code == 'TXVALUE': - stack.append(tx.value) - elif code == 'TXFEE': - stack.append(tx.fee) - elif code == 'TXDATAN': - stack.append(len(tx.data)) - elif code == 'TXDATA': - x, = stack_pop(1) - stack.append(0 if x >= len(tx.data) else tx.data[x]) - elif code == 'BLK_PREVHASH': - stack.append(decode(block.prevhash,256)) - elif code == 'BLK_COINBASE': - stack.append(decode(block.coinbase,160)) - elif code == 'BLK_TIMESTAMP': - stack.append(block.timestamp) - elif code == 'BLK_NUMBER': - stack.append(block.number) - elif code == 'BLK_DIFFICULTY': - stack.append(block.difficulty) - elif code == 'SHA256': - L = stack_pop(1) - hdataitems = stack_pop(math.ceil(L / 32.0)) - hdata = ''.join([encode(x,256,32) for x in hdataitems])[:L] - stack.append(decode(hashlib.sha256(hdata).digest(),256)) - elif code == 'RIPEMD-160': - L = stack_pop(1) - hdataitems = stack_pop(math.ceil(L / 32.0)) - hdata = ''.join([encode(x,256,32) for x in hdataitems])[:L] - stack.append(decode(hashlib.new('ripemd160',hdata).digest(),256)) - elif code == 'ECMUL': - n,x,y = stack_pop(3) - # Point at infinity - if x == 0 and y == 0: - stack.extend([0,0]) - # Point not on curve, coerce to infinity - elif x >= P or y >= P or (x ** 3 + 7 - y ** 2) % P != 0: - stack.extend([0,0]) - # Legitimate point - else: - x2,y2 = base10_multiply((x,y),n) - stack.extend([x2,y2]) - elif code == 'ECADD': - x1,y1,x2,y2 = stack_pop(4) - # Invalid point 1 - if x1 >= P or y1 >= P or (x1 ** 3 + 7 - y1 ** 2) % P != 0: - stack.extend([0,0]) - # Invalid point 2 - elif x2 >= P or y2 >= P or (x2 ** 3 + 7 - y2 ** 2) % P != 0: - stack.extend([0,0]) - # Legitimate points - else: - x3,y3 = base10_add((x1,y1),(x2,y2)) - stack.extend([x3,y3]) - elif code == 'ECSIGN': - k,h = stack_pop(2) - v,r,s = ecdsa_raw_sign(h,k) - stack.extend([v,r,s]) - elif code == 'ECRECOVER': - h,v,r,s = stack_pop(4) - x,y = ecdsa_raw_recover((v,r,s),h) - stack.extend([x,y]) - elif code == 'PUSH': - stack.append(contract.get(encode(index + 1,256,32))) - index += 1 - elif code == 'POP': - stack_pop(1) - elif code == 'DUP': - x, = stack_pop(1) - stack.extend([x,x]) - elif code == 'DUPN': - arr = stack_pop(contract.get(encode(index + 1,256,32))) - arr.append(arr[0]) - stack.extend(arr) - index += 1 - elif code == 'SWAP': - x,y = stack_pop(2) - stack.extend([y,x]) - elif code == 'SWAPN': - arr = stack_pop(contract.get(encode(index + 1,256,32))) - arr.append(arr[0]) - arr.pop(0) - stack.extend(arr) - index += 1 - elif code == 'LOAD': - stack.append(contract.get(encode(stack_pop(1)[0],256,32))) - elif code == 'STORE': - x,y = stack_pop(2) - if exit: break - contract.update(encode(x,256,32),y) - elif code == 'JMP': - index = stack_pop(1)[0] - elif code == 'JMPI': - newpos,c = stack_pop(2) - if c != 0: index = newpos - elif code == 'IND': - stack.append(index) - elif code == 'EXTRO': - ind,addr = stack_pop(2) - stack.push(block.get_contract(encode(addr,256,20)).get(encode(ind,256,32))) - elif code == 'BALANCE': - stack.push(block.get_balance(encode(stack_pop(1)[0],256,20))) - elif code == 'MKTX': - datan,fee,value,to = stack_pop(4) - if exit: - break - elif (value + fee) > block.get_balance(address): - break - else: - data = stack_pop(datan) - tx = Transaction(0,encode(to,256,20),value,fee,data) - tx.sender = address - transaction_list.insert(0,tx) - elif code == 'SUICIDE': - sz = contract.get_size() - negfee = -sz * getfee("memoryfee") - toaddress = encode(stack_pop(1)[0],256,20) - block.pay_fee(toaddress,negfee,False) - contract.root = '' - break - if exit: break - block.update_contract(address,contract) diff --git a/rlp.py b/rlp.py deleted file mode 100644 index e35f111941..0000000000 --- a/rlp.py +++ /dev/null @@ -1,86 +0,0 @@ -def binary_length(n): - if n == 0: return 0 - else: return 1 + binary_length(n / 256) - -def to_binary_array(n,L=None): - if L is None: L = binary_length(n) - if n == 0: return [] - else: - x = to_binary_array(n / 256) - x.append(n % 256) - return x - -def to_binary(n,L=None): return ''.join([chr(x) for x in to_binary_array(n,L)]) - -def from_binary(b): - if len(b) == 0: return 0 - else: return from_binary(b[:-1]) * 256 + ord(b[-1]) - -def __decode(s,pos=0): - if not s: - return (None, 0) - else: - fchar = ord(s[pos]) - if fchar < 24: - return (ord(s[pos]), pos+1) - elif fchar < 56: - b = ord(s[pos]) - 23 - return (from_binary(s[pos+1:pos+1+b]), pos+1+b) - elif fchar < 64: - b = ord(s[pos]) - 55 - b2 = from_binary(s[pos+1:pos+1+b]) - return (from_binary(s[pos+1+b:pos+1+b+b2]), pos+1+b+b2) - elif fchar < 120: - b = ord(s[pos]) - 64 - return (s[pos+1:pos+1+b], pos+1+b) - elif fchar < 128: - b = ord(s[pos]) - 119 - b2 = from_binary(s[pos+1:pos+1+b]) - return (s[pos+1+b:pos+1+b+b2], pos+1+b+b2) - elif fchar < 184: - b = ord(s[pos]) - 128 - o, pos = [], pos+1 - for i in range(b): - obj, pos = __decode(s,pos) - o.append(obj) - return (o,pos) - elif fchar < 192: - b = ord(s[pos]) - 183 - b2 = from_binary(s[pos+1:pos+1+b]) - o, pos = [], pos+1+b - for i in range(b): - obj, pos = __decode(s,pos) - o.append(obj) - return (o,pos) - else: - raise Exception("byte not supported: "+fchar) - -def decode(s): return __decode(s)[0] - -def encode(s): - if isinstance(s,(int,long)): - if s < 0: - raise Exception("can't handle negative ints") - elif s >= 0 and s < 24: - return chr(s) - elif s < 2**256: - b = to_binary(s) - return chr(len(b) + 23) + b - else: - b = to_binary(s) - b2 = to_binary(len(b)) - return chr(len(b2) + 55) + b2 + b - elif isinstance(s,(str,unicode)): - if len(s) < 56: - return chr(len(s) + 64) + str(s) - else: - b2 = to_binary(len(s)) - return chr(len(b2) + 119) + b2 + str(s) - elif isinstance(s,list): - if len(s) < 56: - return chr(len(s) + 128) + ''.join([encode(x) for x in s]) - else: - b2 = to_binary(len(s)) - return chr(len(b2) + 183) + b2 + ''.join([encode(x) for x in s]) - else: - raise Exception("Encoding for "+s+" not yet implemented") diff --git a/runtest.py b/runtest.py deleted file mode 100644 index 5ef92f5d57..0000000000 --- a/runtest.py +++ /dev/null @@ -1,81 +0,0 @@ -import json, sys, os -import rlp, trie -import random - -testdir = sys.argv[1] if len(sys.argv) >= 2 else '../tests' - -rlpdata = json.loads(open(os.path.join(testdir,'rlptest.txt')).read()) -for x,y in rlpdata: - yprime = rlp.encode(x).encode('hex') - if yprime != y: print "RLPEncode Mismatch: ",x,y,yprime - xprime = rlp.decode(y.decode('hex')) - jx, jxprime = json.dumps(x), json.dumps(xprime) - if jx != jxprime: print "RLPDecode Mismatch: ",jx,jxprime,y - -hexencodedata = json.loads(open(os.path.join(testdir,'hexencodetest.txt')).read()) - -for x,y in hexencodedata: - yprime = trie.hexarraykey_to_bin(x).encode('hex') - if yprime != y: print "HexEncode Mismatch: ",x,y,yprime - xprime = trie.bin_to_hexarraykey(y.decode('hex')) - jx,jxprime = json.dumps(x), json.dumps(xprime) - if jx != jxprime: print "HexDecode Mismatch: ",jx,jxprime,y - -triedata = json.loads(open(os.path.join(testdir,'trietest.txt')).read()) - -for x,y in triedata: - t0 = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) - for k in x: - t0.update(k,x[k]) - if t0.root.encode('hex') != y: - print "Mismatch with adds only" - continue - t = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) - dummies, reals = [], [] - for k in x: - reals.append([k,x[k]]) - dummies.append(k[:random.randrange(len(k)-1)]) - dummies.append(k+random.choice(dummies)) - dummies.append(k[:random.randrange(len(k)-1)]+random.choice(dummies)) - dummies_to_pop = set([]) - i = 0 - ops = [] - mp = {} - success = [True] - def update(k,v): - t.update(k,v) - if v == '' and k in mp: del mp[k] - else: mp[k] = v - ops.append([k,v,t.root.encode('hex')]) - tn = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) - for k in mp: - tn.update(k,mp[k]) - if tn.root != t.root: - print "Mismatch: " - for op in ops: print op - success[0] = False - while i < len(reals): - s = random.randrange(3) - if s == 0: - update(reals[i][0],reals[i][1]) - i += 1 - elif s == 1: - k,v = random.choice(dummies), random.choice(dummies) - update(k,v) - dummies_to_pop.add(k) - elif s == 2: - if len(dummies_to_pop) > 0: - k = random.choice(list(dummies_to_pop)) - update(k,'') - dummies_to_pop.remove(k) - if not success[0]: - break - if not success[0]: - continue - i = len(reals) * 2 - while len(dummies_to_pop) > 0: - k = random.choice(list(dummies_to_pop)) - update(k,'') - dummies_to_pop.remove(k) - if not success[0]: - break diff --git a/transactions.py b/transactions.py deleted file mode 100644 index 1cd1d74512..0000000000 --- a/transactions.py +++ /dev/null @@ -1,47 +0,0 @@ -from pybitcointools import * -import rlp -import re - -class Transaction(): - def __init__(*args): - self = args[0] - if len(args) == 2: - self.parse(args[1]) - else: - self.nonce = args[1] - self.to = args[2] - self.value = args[3] - self.fee = args[4] - self.data = args[5] - - def parse(self,data): - if re.match('^[0-9a-fA-F]*$',data): - data = data.decode('hex') - o = rlp.decode(data) - self.nonce = o[0] - self.to = o[1] - self.value = o[2] - self.fee = o[3] - self.data = o[4] - self.v = o[5] - self.r = o[6] - self.s = o[7] - rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) - pub = encode_pubkey(ecdsa_raw_recover(rawhash,(self.v,self.r,self.s)),'bin') - self.sender = bin_sha256(pub[1:])[-20:] - return self - - def sign(self,key): - rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) - self.v,self.r,self.s = ecdsa_raw_sign(rawhash,key) - self.sender = bin_sha256(privtopub(key)[1:])[-20:] - return self - - def serialize(self): - return rlp.encode([self.nonce, self.to, self.value, self.fee, self.data, self.v, self.r, self.s]) - - def hex_serialize(self): - return self.serialize().encode('hex') - - def hash(self): - return bin_sha256(self.serialize()) diff --git a/trie.py b/trie.py deleted file mode 100644 index 2aa9a3d0fd..0000000000 --- a/trie.py +++ /dev/null @@ -1,233 +0,0 @@ -import leveldb -import rlp -from sha3 import sha3_256 - -def sha3(x): return sha3_256(x).digest() - -class DB(): - def __init__(self,dbfile): self.db = leveldb.LevelDB(dbfile) - def get(self,key): - try: return self.db.Get(key) - except: return '' - def put(self,key,value): return self.db.Put(key,value) - def delete(self,key): return self.db.Delete(key) - -def hexarraykey_to_bin(key): - term = 1 if key[-1] == 16 else 0 - if term: key = key[:-1] - oddlen = len(key) % 2 - flags = 2 * term + oddlen - if oddlen: key = [flags] + key - else: key = [flags,0] + key - o = '' - for i in range(0,len(key),2): - o += chr(16 * key[i] + key[i+1]) - return o - -def bin_to_hexarraykey(bindata): - o = ['0123456789abcdef'.find(x) for x in bindata.encode('hex')] - if o[0] >= 2: o.append(16) - if o[0] % 2 == 1: o = o[1:] - else: o = o[2:] - return o - -databases = {} - -class Trie(): - def __init__(self,dbfile,root='',debug=False): - self.root = root - self.debug = debug - if dbfile not in databases: - databases[dbfile] = DB(dbfile) - self.db = databases[dbfile] - - def __get_state(self,node,key): - if self.debug: print 'nk',node.encode('hex'),key - if len(key) == 0 or not node: - return node - curnode = self.lookup(node) - if self.debug: print 'cn', curnode - if not curnode: - raise Exception("node not found in database") - elif len(curnode) == 2: - (k2,v2) = curnode - k2 = bin_to_hexarraykey(k2) - if len(key) >= len(k2) and k2 == key[:len(k2)]: - return self.__get_state(v2,key[len(k2):]) - else: - return '' - elif len(curnode) == 17: - return self.__get_state(curnode[key[0]],key[1:]) - - def __put(self,node,root=False): - rlpnode = rlp.encode(node) - if len(rlpnode) >= 32: - h = sha3(rlpnode) - self.db.put(h,rlpnode) - else: - h = rlpnode if root else node - return h - - def lookup(self,node): - if not isinstance(node,(str,unicode)): return node - elif len(node) == 0: return node - elif len(node) < 32: return rlp.decode(node) - else: return rlp.decode(self.db.get(node)) - - def __update_state(self,node,key,value): - if value != '': return self.__insert_state(node,key,value) - else: return self.__delete_state(node,key) - - def __insert_state(self,node,key,value): - if self.debug: print 'ins', node.encode('hex'), key - if len(key) == 0: - return value - else: - if not node: - newnode = [ hexarraykey_to_bin(key), value ] - return self.__put(newnode) - curnode = self.lookup(node) - if self.debug: print 'icn', curnode - if not curnode: - raise Exception("node not found in database") - if len(curnode) == 2: - (k2, v2) = curnode - k2 = bin_to_hexarraykey(k2) - if key == k2: - newnode = [ hexarraykey_to_bin(key), value ] - return self.__put(newnode) - else: - i = 0 - while key[:i+1] == k2[:i+1] and i < len(k2): i += 1 - if i == len(k2): - newhash3 = self.__insert_state(v2,key[i:],value) - else: - newnode1 = self.__insert_state('',key[i+1:],value) - newnode2 = self.__insert_state('',k2[i+1:],v2) - newnode3 = [ '' ] * 17 - newnode3[key[i]] = newnode1 - newnode3[k2[i]] = newnode2 - newhash3 = self.__put(newnode3) - if i == 0: - return newhash3 - else: - newnode4 = [ hexarraykey_to_bin(key[:i]), newhash3 ] - return self.__put(newnode4) - else: - newnode = [ curnode[i] for i in range(17) ] - newnode[key[0]] = self.__insert_state(curnode[key[0]],key[1:],value) - return self.__put(newnode) - - def __delete_state(self,node,key): - if self.debug: print 'dnk', node.encode('hex'), key - if len(key) == 0 or not node: - return '' - else: - curnode = self.lookup(node) - if not curnode: - raise Exception("node not found in database") - if self.debug: print 'dcn', curnode - if len(curnode) == 2: - (k2, v2) = curnode - k2 = bin_to_hexarraykey(k2) - if key == k2: - return '' - elif key[:len(k2)] == k2: - newhash = self.__delete_state(v2,key[len(k2):]) - childnode = self.lookup(newhash) - if len(childnode) == 2: - newkey = k2 + bin_to_hexarraykey(childnode[0]) - newnode = [ hexarraykey_to_bin(newkey), childnode[1] ] - else: - newnode = [ curnode[0], newhash ] - return self.__put(newnode) - else: return node - else: - newnode = [ curnode[i] for i in range(17) ] - newnode[key[0]] = self.__delete_state(newnode[key[0]],key[1:]) - onlynode = -1 - for i in range(17): - if newnode[i]: - if onlynode == -1: onlynode = i - else: onlynode = -2 - if onlynode == 16: - newnode2 = [ hexarraykey_to_bin([16]), newnode[onlynode] ] - elif onlynode >= 0: - childnode = self.lookup(newnode[onlynode]) - if not childnode: - raise Exception("?????") - if len(childnode) == 17: - newnode2 = [ hexarraykey_to_bin([onlynode]), newnode[onlynode] ] - elif len(childnode) == 2: - newkey = [onlynode] + bin_to_hexarraykey(childnode[0]) - newnode2 = [ hexarraykey_to_bin(newkey), childnode[1] ] - else: - newnode2 = newnode - return self.__put(newnode2) - - def __get_size(self,node): - if not node: return 0 - curnode = self.lookup(node) - if not curnode: - raise Exception("node not found in database") - if len(curnode) == 2: - key = hexarraykey_to_bin(curnode[0]) - if key[-1] == 16: return 1 - else: return self.__get_size(curnode[1]) - elif len(curnode) == 17: - total = 0 - for i in range(16): - total += self.__get_size(curnode[i]) - if curnode[16]: total += 1 - return total - - def __to_dict(self,node): - if not node: return {} - curnode = self.lookup(node) - if not curnode: - raise Exception("node not found in database") - if len(curnode) == 2: - lkey = bin_to_hexarraykey(curnode[0]) - o = {} - if lkey[-1] == 16: - o[curnode[0]] = curnode[1] - else: - d = self.__to_dict(curnode[1]) - for v in d: - subkey = bin_to_hexarraykey(v) - totalkey = hexarraykey_to_bin(lkey+subkey) - o[totalkey] = d[v] - return o - elif len(curnode) == 17: - o = {} - for i in range(16): - d = self.__to_dict(curnode[i]) - for v in d: - subkey = bin_to_hexarraykey(v) - totalkey = hexarraykey_to_bin([i] + subkey) - o[totalkey] = d[v] - if curnode[16]: o[chr(16)] = curnode[16] - return o - else: - raise Exception("bad curnode! "+curnode) - - def to_dict(self,as_hex=False): - d = self.__to_dict(self.root) - o = {} - for v in d: - v2 = ''.join(['0123456789abcdef'[x] for x in bin_to_hexarraykey(v)[:-1]]) - if not as_hex: v2 = v2.decode('hex') - o[v2] = d[v] - return o - - def get(self,key): - key2 = ['0123456789abcdef'.find(x) for x in str(key).encode('hex')] + [16] - return self.__get_state(self.root,key2) - - def get_size(self): return self.__get_size(self.root) - - def update(self,key,value): - if not isinstance(key,(str,unicode)) or not isinstance(value,(str,unicode)): - raise Exception("Key and value must be strings") - key2 = ['0123456789abcdef'.find(x) for x in str(key).encode('hex')] + [16] - self.root = self.__update_state(self.root,key2,str(value)) diff --git a/trietest.py b/trietest.py deleted file mode 100644 index d427b1e4f9..0000000000 --- a/trietest.py +++ /dev/null @@ -1,25 +0,0 @@ -from trie import Trie -import random - -def genkey(): - L = random.randrange(30) - if random.randrange(5) == 0: return '' - return ''.join([random.choice('1234579qetyiasdfghjklzxcvbnm') for x in range(L)]) - -t = Trie('/tmp/'+genkey()) - -def trie_test(): - o = {} - for i in range(60): - key, value = genkey(), genkey() - if value: print "setting key: '"+key+"', value: '"+value+"'" - else: print "deleting key: '"+key+"'" - o[key] = value - t.update(key,value) - for k in o.keys(): - v1 = o[k] - v2 = t.get(k) - print v1,v2 - if v1 != v2: raise Exception("incorrect!") - -trie_test()