diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 300b8eef5b..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "tests"] - path = tests/testdata - url = https://github.com/ethereum/tests \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4536056443..ce9b9c7a2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -# Build XDC in a stock Go builder container FROM golang:1.10-alpine as builder RUN apk add --no-cache make gcc musl-dev linux-headers @@ -8,7 +7,7 @@ RUN cd /XDCchain && make XDC FROM alpine:latest -LABEL maintainer="admin@xinfin.org" +LABEL maintainer="anil@xinfin.org" WORKDIR /XDCchain @@ -22,4 +21,3 @@ EXPOSE 30303 ENTRYPOINT ["/usr/local/bin/XDC"] CMD ["--help"] - diff --git a/Dockerfile.bootnode b/Dockerfile.bootnode index 6a3ce65c64..9070495f6a 100644 --- a/Dockerfile.bootnode +++ b/Dockerfile.bootnode @@ -9,7 +9,7 @@ RUN chmod +x /XDCchain/build/bin/bootnode FROM alpine:latest -LABEL maintainer="admin@xinfin.org" +LABEL maintainer="anil@xinfin.org" WORKDIR /XDCchain @@ -21,4 +21,4 @@ EXPOSE 30301 ENTRYPOINT ["./entrypoint.sh"] -CMD ["-verbosity", "6", "-nodekey", "bootnode.key", "--addr", ":30301" \ No newline at end of file +CMD ["-verbosity", "6", "-nodekey", "bootnode.key", "--addr", ":30301"] diff --git a/Dockerfile.node b/Dockerfile.node index eaea4c6163..581f750fa6 100644 --- a/Dockerfile.node +++ b/Dockerfile.node @@ -1,22 +1,22 @@ - FROM golang:1.10-alpine as builder +FROM golang:1.10-alpine as builder - RUN apk add --no-cache make gcc musl-dev linux-headers +RUN apk add --no-cache make gcc musl-dev linux-headers - ADD . /XDCchain +ADD . /XDChain - RUN cd /XDCchain \ - && make XDC \ +RUN cd /XDChain \ + && make XDC\ && chmod +x /XDCchain/build/bin/XDC - FROM alpine:latest +FROM alpine:latest - LABEL maintainer="admin@xinfin.org" +LABEL maintainer="anil@xinfin.org" - WORKDIR /XDCchain +WORKDIR /XDChain - COPY --from=builder /XDCchain/build/bin/XDC /usr/local/bin/XDC +COPY --from=builder /XDCchain/build/bin/XDC /usr/local/bin/XDC - ENV IDENTITY '' +ENV IDENTITY '' ENV PASSWORD '' ENV PRIVATE_KEY '' ENV BOOTNODES '' @@ -29,11 +29,11 @@ ENV NETSTATS_HOST 'netstats-server' ENV NETSTATS_PORT '3000' ENV ANNOUNCE_TXS '' - RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates - COPY docker/XDCchain ./ +COPY docker/XDChain ./ COPY genesis/ ./ - EXPOSE 8545 8546 30303 30303/udp +EXPOSE 8545 8546 30303 30303/udp - ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["./entrypoint.sh"] diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 39ff5d83c6..0000000000 --- a/circle.yml +++ /dev/null @@ -1,32 +0,0 @@ -machine: - services: - - docker - -dependencies: - cache_directories: - - "~/.ethash" # Cache the ethash DAG generated by hive for consecutive builds - - "~/.docker" # Cache all docker images manually to avoid lengthy rebuilds - override: - # Restore all previously cached docker images - - mkdir -p ~/.docker - - for img in `ls ~/.docker`; do docker load -i ~/.docker/$img; done - - # Pull in and hive, restore cached ethash DAGs and do a dry run - - go get -u github.com/karalabe/hive - - (cd ~/.go_workspace/src/github.com/karalabe/hive && mkdir -p workspace/ethash/ ~/.ethash) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && cp -r ~/.ethash/. workspace/ethash/) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=NONE --test=. --sim=. --loglevel=6) - - # Cache all the docker images and the ethash DAGs - - for img in `docker images | grep -v "^" | tail -n +2 | awk '{print $1}'`; do docker save $img > ~/.docker/`echo $img | tr '/' ':'`.tar; done - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/ethash/. ~/.ethash - -test: - override: - # Build Geth and move into a known folder - - make geth - - cp ./build/bin/geth $HOME/geth - - # Run hive and move all generated logs into the public artifacts folder - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=go-ethereum:local --override=$HOME/geth --test=. --sim=.) - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/logs/* $CIRCLE_ARTIFACTS diff --git a/cmd/XDC/genesis_test.go b/cmd/XDC/genesis_test.go deleted file mode 100644 index 9044e6e8ef..0000000000 --- a/cmd/XDC/genesis_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" -) - -var customGenesisTests = []struct { - genesis string - query string - result string -}{ - // Plain genesis file without anything extra - { - genesis: `{ - "alloc" : {}, - "coinbase" : "0x0000000000000000000000000000000000000000", - "difficulty" : "0x20000", - "extraData" : "", - "gasLimit" : "0x2fefd8", - "nonce" : "0x0000000000000042", - "mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000", - "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", - "timestamp" : "0x00" - }`, - query: "eth.getBlock(0).nonce", - result: "0x0000000000000042", - }, - // Genesis file with an empty chain configuration (ensure missing fields work) - { - genesis: `{ - "alloc" : {}, - "coinbase" : "0x0000000000000000000000000000000000000000", - "difficulty" : "0x20000", - "extraData" : "", - "gasLimit" : "0x2fefd8", - "nonce" : "0x0000000000000042", - "mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000", - "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", - "timestamp" : "0x00", - "config" : {} - }`, - query: "eth.getBlock(0).nonce", - result: "0x0000000000000042", - }, - // Genesis file with specific chain configurations - { - genesis: `{ - "alloc" : {}, - "coinbase" : "0x0000000000000000000000000000000000000000", - "difficulty" : "0x20000", - "extraData" : "", - "gasLimit" : "0x2fefd8", - "nonce" : "0x0000000000000042", - "mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000", - "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000", - "timestamp" : "0x00", - "config" : { - "homesteadBlock" : 314, - "daoForkBlock" : 141, - "daoForkSupport" : true - }, - }`, - query: "eth.getBlock(0).nonce", - result: "0x0000000000000042", - }, -} - -// Tests that initializing XDC with a custom genesis block and chain definitions -// work properly. -func TestCustomGenesis(t *testing.T) { - for i, tt := range customGenesisTests { - // Create a temporary data directory to use and inspect later - datadir := tmpdir(t) - defer os.RemoveAll(datadir) - - // Initialize the data directory with the custom genesis block - json := filepath.Join(datadir, "genesis.json") - if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil { - t.Fatalf("test %d: failed to write genesis file: %v", i, err) - } - runXDC(t, "--datadir", datadir, "init", json).WaitExit() - - // Query the custom genesis block - XDC := runXDC(t, - "--datadir", datadir, "--maxpeers", "0", "--port", "0", - "--nodiscover", "--nat", "none", "--ipcdisable", - "--exec", tt.query, "console") - XDC.ExpectRegexp(tt.result) - XDC.ExpectExit() - } -} diff --git a/cmd/XDC/main.go b/cmd/XDC/main.go index 0367dd578f..0d531da427 100644 --- a/cmd/XDC/main.go +++ b/cmd/XDC/main.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/XDPoS" "github.com/ethereum/go-ethereum/console" "github.com/ethereum/go-ethereum/core" @@ -123,6 +124,7 @@ var ( configFileFlag, utils.AnnounceTxsFlag, utils.StoreRewardFlag, + utils.RollbackFlag, } rpcFlags = []cli.Flag{ @@ -291,9 +293,18 @@ func startNode(ctx *cli.Context, stack *node.Node, cfg XDCConfig) { if _, ok := ethereum.Engine().(*XDPoS.XDPoS); ok { go func() { started := false - ok, err := ethereum.ValidateMasternode() - if err != nil { - utils.Fatalf("Can't verify masternode permission: %v", err) + ok := false + var err error + if common.IsTestnet { + ok, err = ethereum.ValidateMasternodeTestnet() + if err != nil { + utils.Fatalf("Can't verify masternode permission: %v", err) + } + } else { + ok, err = ethereum.ValidateMasternode() + if err != nil { + utils.Fatalf("Can't verify masternode permission: %v", err) + } } if ok { log.Info("Masternode found. Enabling staking mode...") @@ -317,9 +328,16 @@ func startNode(ctx *cli.Context, stack *node.Node, cfg XDCConfig) { defer close(core.CheckpointCh) for range core.CheckpointCh { log.Info("Checkpoint!!! It's time to reconcile node's state...") - ok, err := ethereum.ValidateMasternode() - if err != nil { - utils.Fatalf("Can't verify masternode permission: %v", err) + if common.IsTestnet { + ok, err = ethereum.ValidateMasternodeTestnet() + if err != nil { + utils.Fatalf("Can't verify masternode permission: %v", err) + } + } else { + ok, err = ethereum.ValidateMasternode() + if err != nil { + utils.Fatalf("Can't verify masternode permission: %v", err) + } } if !ok { if started { diff --git a/cmd/XDCclean/main.go b/cmd/XDCclean/main.go new file mode 100644 index 0000000000..bd41f9234c --- /dev/null +++ b/cmd/XDCclean/main.go @@ -0,0 +1,270 @@ +package main + +import ( + "flag" + "fmt" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/hashicorp/golang-lru" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" + "os" + "os/signal" + "runtime" + "sync" + "sync/atomic" + "time" +) + +var ( + dir = flag.String("dir", "", "dir to mainet chain data") + cacheSize = flag.Int("size", 1000000, "dir to mainet chain data") +) + +type TrieRoot struct { + trie *trie.SecureTrie + number uint64 +} +type StateNode struct { + node trie.Node + path []byte +} +type ResultProcessNode struct { + index int + number int + newNodes [17]*StateNode + keys [17]*[]byte +} + +var sercureKey = []byte("secure-key-") +var nWorker = runtime.NumCPU() / 2 +var cleanAddress = []common.Address{common.HexToAddress(common.BlockSigners)} +var cache *lru.Cache +var finish = int32(0) +var running = true +var stateRoots = make(chan TrieRoot) + +func main() { + flag.Parse() + lddb, _ := ethdb.NewLDBDatabase(*dir, eth.DefaultConfig.DatabaseCache, utils.MakeDatabaseHandles()) + head := core.GetHeadBlockHash(lddb) + currentHeader := core.GetHeader(lddb, head, core.GetBlockNumber(lddb, head)) + tridb := trie.NewDatabase(lddb) + catchEventInterupt(lddb.LDB()) + cache, _ = lru.New(*cacheSize) + go func() { + for i := uint64(1); i <= currentHeader.Number.Uint64(); i++ { + hash := core.GetCanonicalHash(lddb, i) + root := core.GetHeader(lddb, hash, i).Root + trieRoot, err := trie.NewSecure(root, tridb, 0) + if err != nil { + continue + } + if running { + stateRoots <- TrieRoot{trieRoot, i} + } else { + break + } + } + if running { + close(stateRoots) + } + }() + for trieRoot := range stateRoots { + atomic.StoreInt32(&finish, 1) + if running { + for _, address := range cleanAddress { + enc := trieRoot.trie.Get(address.Bytes()) + var data state.Account + rlp.DecodeBytes(enc, &data) + fmt.Println(time.Now().Format(time.RFC3339), "Start clean state address ", address.Hex(), " at block ", trieRoot.number) + signerRoot, err := resolveHash(data.Root[:], lddb.LDB()) + if err != nil { + fmt.Println(time.Now().Format(time.RFC3339), "Not found clean state address ", address.Hex(), " at block ", trieRoot.number) + continue + } + batch := new(leveldb.Batch) + count := 1 + list := []*StateNode{{node: signerRoot}} + for len(list) > 0 { + newList, total := findNewNodes(list, lddb.LDB(), batch) + count = count + 17*len(newList) + list = removeNodesNil(newList, total) + } + fmt.Println(time.Now().Format(time.RFC3339), "Finish clean state address ", address.Hex(), " at block ", trieRoot.number, " keys ", count) + err = lddb.LDB().Write(batch, nil) + if err != nil { + fmt.Println(time.Now().Format(time.RFC3339), "Write batch leveldb error", err) + os.Exit(1) + } + } + } else { + break + } + atomic.StoreInt32(&finish, 0) + } + fmt.Println(time.Now(), "compact") + lddb.LDB().CompactRange(util.Range{}) + lddb.Close() + fmt.Println(time.Now(), "end") +} + +func removeNodesNil(list [][17]*StateNode, length int) []*StateNode { + results := make([]*StateNode, length) + index := 0 + for _, nodes := range list { + for _, node := range nodes { + if node != nil { + results[index] = node + index++ + } + } + } + return results +} +func catchEventInterupt(db *leveldb.DB) { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for sig := range c { + fmt.Println("catch event interrupt ", sig, running, finish) + running = false + if atomic.LoadInt32(&finish) == 0 { + close(stateRoots) + db.Close() + os.Exit(1) + } + } + }() +} +func resolveHash(n trie.HashNode, db *leveldb.DB) (trie.Node, error) { + if cache.Contains(common.BytesToHash(n)) { + return nil, &trie.MissingNodeError{} + } + enc, err := db.Get(n, nil) + if err != nil || enc == nil { + return nil, &trie.MissingNodeError{} + } + return trie.MustDecodeNode(n, enc, 0), nil +} + +func getAllChilds(n StateNode, db *leveldb.DB) ([17]*StateNode, error) { + childs := [17]*StateNode{} + switch node := n.node.(type) { + case *trie.FullNode: + // Full Node, move to the first non-nil child. + for i := 0; i < len(node.Children); i++ { + child := node.Children[i] + if child != nil { + childNode := child + var err error = nil + if _, ok := child.(trie.HashNode); ok { + childNode, err = resolveHash(child.(trie.HashNode), db) + } + if err == nil { + childs[i] = &StateNode{node: childNode, path: append(n.path, byte(i))} + } else if err != nil { + _, ok := err.(*trie.MissingNodeError) + if !ok { + return childs, err + } + } + } + } + case *trie.ShortNode: + // Short Node, return the pointer singleton child + childNode := node.Val + var err error = nil + if _, ok := node.Val.(trie.HashNode); ok { + childNode, err = resolveHash(node.Val.(trie.HashNode), db) + } + if err == nil { + childs[0] = &StateNode{node: childNode, path: append(n.path, node.Key...)} + } else if err != nil { + _, ok := err.(*trie.MissingNodeError) + if !ok { + return childs, err + } + } + } + return childs, nil +} +func processNodes(node StateNode, db *leveldb.DB) ([17]*StateNode, [17]*[]byte, int) { + hash, _ := node.node.Cache() + commonHash := common.BytesToHash(hash) + newNodes := [17]*StateNode{} + keys := [17]*[]byte{} + number := 0 + if !cache.Contains(commonHash) { + childNodes, err := getAllChilds(node, db) + if err != nil { + fmt.Println("Error when get all childs node : ", common.Bytes2Hex(node.path), err) + os.Exit(1) + } + for i, child := range childNodes { + if child != nil { + if _, ok := child.node.(trie.ValueNode); ok { + buf := append(sercureKey, child.path...) + keys[i] = &buf + } else { + hash, _ := child.node.Cache() + var bytes []byte = hash + keys[i] = &bytes + newNodes[i] = child + number++ + } + } + } + cache.Add(commonHash, true) + } + return newNodes, keys, number +} + +func findNewNodes(nodes []*StateNode, db *leveldb.DB, batchlvdb *leveldb.Batch) ([][17]*StateNode, int) { + length := len(nodes) + chunkSize := length / nWorker + if len(nodes)%nWorker != 0 { + chunkSize++ + } + childNodes := make([][17]*StateNode, length) + results := make(chan ResultProcessNode) + wg := sync.WaitGroup{} + wg.Add(length) + for i := 0; i < nWorker; i++ { + from := i * chunkSize + to := from + chunkSize + if to > length { + to = length + } + go func(from int, to int) { + for j := from; j < to; j++ { + childs, keys, number := processNodes(*nodes[j], db) + go func(result ResultProcessNode) { + results <- result + }(ResultProcessNode{j, number, childs, keys}) + } + }(from, to) + } + total := 0 + go func() { + for result := range results { + childNodes[result.index] = result.newNodes + total = total + result.number + for _, key := range result.keys { + if key != nil { + batchlvdb.Delete(*key) + } + } + wg.Done() + } + }() + wg.Wait() + close(results) + return childNodes, total +} diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 05150f2a3b..3832b247f8 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -40,7 +40,9 @@ var dashboardContent = ` + {{.NetworkTitle}}: Ethereum Testnet + @@ -62,6 +64,7 @@ var dashboardContent = ` } +
@@ -260,11 +263,13 @@ var dashboardContent = ` 
 Enodes bootnodes = new Enodes();{{range .Bootnodes}}
 bootnodes.append(new Enode("{{.}}"));{{end}}
+
 NodeConfig config = new NodeConfig();
 config.setBootstrapNodes(bootnodes);
 config.setEthereumNetworkID({{.NetworkID}});
 config.setEthereumGenesis(genesis);{{if .Ethstats}}
 config.setEthereumNetStats("{{.Ethstats}}");{{end}}
+
 Node node = new Node(getFilesDir() + "/.{{.Network}}", config);
 node.start();
@@ -288,13 +293,16 @@ node.start(); 
import Geth
 var error: NSError?
+
 let bootnodes = GethNewEnodesEmpty(){{range .Bootnodes}}
 bootnodes?.append(GethNewEnode("{{.}}", &error)){{end}}
+
 let config = GethNewNodeConfig()
 config?.setBootstrapNodes(bootnodes)
 config?.setEthereumNetworkID({{.NetworkID}})
 config?.setEthereumGenesis(genesis){{if .Ethstats}}
 config?.setEthereumNetStats("{{.Ethstats}}"){{end}}
+
 let datadir = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
 let node = GethNewNode(datadir + "/.{{.Network}}", config, &error);
 try! node?.start();
@@ -428,12 +436,14 @@ try! node?.start();
+