diff --git a/.gitignore b/.gitignore index 21dbd28c56..9ec244c6d2 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,10 @@ Godeps/_workspace/bin # travis profile.tmp profile.cov + +# vagrant +.vagrant + +abigen + +geth diff --git a/accounts/abi/bind/ext.go b/accounts/abi/bind/ext.go index 3b9ffb3edd..0e88e37ee6 100644 --- a/accounts/abi/bind/ext.go +++ b/accounts/abi/bind/ext.go @@ -31,9 +31,9 @@ func DefaultDeployOptions() *DeployOptions { // implemented by eth.APIBackend type Backend interface { - GetTxReceipt(txhash common.Hash) *types.Receipt - CodeAt(address common.Address) string - BalanceAt(address common.Address) *big.Int + GetTxReceipt(txhash common.Hash) (map[string]interface{}, error) + BalanceAt(address common.Address) (*big.Int, error) + CodeAt(address common.Address) (string, error) ContractBackend } @@ -64,7 +64,7 @@ DEPLOY: receiptQueryTimer = time.NewTimer(0).C case <-receiptQueryTimer: - receipt := backend.GetTxReceipt(txhash) + receipt, _ := backend.GetTxReceipt(txhash) receiptQueries++ if receipt == nil { if receiptQueries == opt.MaxReceiptQueryAttempts { @@ -80,7 +80,7 @@ DEPLOY: continue DEPLOY } - contractAddr = receipt.ContractAddress + contractAddr = receipt["contractAddress"].(common.Address) glog.V(logger.Detail).Infof("new chequebook contract mined at %v (owner: %v)", contractAddr.Hex(), deployTransactor.From.Hex()) <-time.NewTimer(opt.ConfirmationInterval).C err = Validate(contractAddr, contractCode, backend) @@ -101,7 +101,10 @@ func Validate(contractAddr common.Address, expCode string, backend Backend) (err if (contractAddr == common.Address{}) { return fmt.Errorf("zero address") } - code := backend.CodeAt(contractAddr) + code, err := backend.CodeAt(contractAddr) + if err != nil { + return err + } if len(expCode) > 0 && code != expCode { return fmt.Errorf("incorrect code %v:\n%v\n%v", contractAddr.Hex(), code, expCode) } diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 67225c701d..7fea16a25e 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -63,7 +63,7 @@ import a private key into a new account. It supports interactive mode, when you are prompted for password as well as non-interactive mode where passwords are supplied via a given password file. Non-interactive mode is only meant for scripted use on test networks or known -safe environ>>>ments. +safe environments. Make sure you remember the password you gave when creating a new account (with either new or import). Without it you are not able to unlock your account. @@ -119,7 +119,7 @@ password to file or expose in any other way. Description: ` ethereum account update
->>>>>>>>>>>> + Update an existing account. The account is saved in the newest version in encrypted format, you are prompted @@ -175,7 +175,7 @@ func accountList(ctx *cli.Context) error { return nil } -// tries unlocking the specifiedqqa few times. +// tries unlocking the specified account a few times. func unlockAccount(ctx *cli.Context, accman *accounts.Manager, address string, i int, passwords []string) (accounts.Account, string) { account, err := utils.MakeAddress(accman, address) if err != nil { @@ -203,7 +203,7 @@ func unlockAccount(ctx *cli.Context, accman *accounts.Manager, address string, i return accounts.Account{}, "" } -// getPassPhrase retrieves the password associated with an account, either fetched +// getPassPhrase retrieves the passwor associated with an account, either fetched // from a list of preloaded passphrases, or requested interactively from the user. func getPassPhrase(prompt string, confirmation bool, i int, passwords []string) string { // If a list of passwords was supplied, retrieve from them @@ -262,7 +262,7 @@ func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrErro // accountCreate creates a new account into the keystore defined by the CLI flags. func accountCreate(ctx *cli.Context) error { accman := utils.MakeAccountManager(ctx) - password := utils.GetPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) + password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) account, err := accman.NewAccount(password) if err != nil { @@ -280,8 +280,8 @@ func accountUpdate(ctx *cli.Context) error { } accman := utils.MakeAccountManager(ctx) - account, oldPassword := utils.UnlockAccount(accman, ctx.Args().First(), 0, nil) - newPassword := utils.GetPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil) + account, oldPassword := unlockAccount(ctx, accman, ctx.Args().First(), 0, nil) + newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil) if err := accman.Update(account, oldPassword, newPassword); err != nil { utils.Fatalf("Could not update the account: %v", err) } @@ -299,7 +299,7 @@ func importWallet(ctx *cli.Context) error { } accman := utils.MakeAccountManager(ctx) - passphrase := utils.GetPassPhrase("", false, 0, utils.MakePasswordList(ctx)) + passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx)) acct, err := accman.ImportPreSaleKey(keyJson, passphrase) if err != nil { @@ -319,7 +319,7 @@ func accountImport(ctx *cli.Context) error { utils.Fatalf("Failed to load the private key: %v", err) } accman := utils.MakeAccountManager(ctx) - passphrase := utils.GetPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) + passphrase := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) acct, err := accman.ImportECDSA(key, passphrase) if err != nil { utils.Fatalf("Could not create the account: %v", err) diff --git a/cmd/geth/js.go b/cmd/geth/js.go deleted file mode 100644 index 178fc76c94..0000000000 --- a/cmd/geth/js.go +++ /dev/null @@ -1,424 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "fmt" - "math/big" - "os" - "os/signal" - "path/filepath" - "regexp" - "sort" - "strings" - - "github.com/codegangsta/cli" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/internal/web3ext" - re "github.com/ethereum/go-ethereum/jsre" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rpc" - "github.com/peterh/liner" - "github.com/robertkrimen/otto" -) - -var ( - passwordRegexp = regexp.MustCompile("personal.[nu]") - leadingSpace = regexp.MustCompile("^ ") - onlyws = regexp.MustCompile("^\\s*$") - exit = regexp.MustCompile("^\\s*exit\\s*;*\\s*$") -) - -type jsre struct { - re *re.JSRE - stack *node.Node - wait chan *big.Int - ps1 string - atexit func() - corsDomain string - client rpc.Client -} - -func makeCompleter(re *jsre) liner.WordCompleter { - return func(line string, pos int) (head string, completions []string, tail string) { - if len(line) == 0 || pos == 0 { - return "", nil, "" - } - // chuck data to relevant part for autocompletion, e.g. in case of nested lines eth.getBalance(eth.coinb - i := 0 - for i = pos - 1; i > 0; i-- { - if line[i] == '.' || (line[i] >= 'a' && line[i] <= 'z') || (line[i] >= 'A' && line[i] <= 'Z') { - continue - } - if i >= 3 && line[i] == '3' && line[i-3] == 'w' && line[i-2] == 'e' && line[i-1] == 'b' { - continue - } - i += 1 - break - } - return line[:i], re.re.CompleteKeywords(line[i:pos]), line[pos:] - } -} - -func newLightweightJSRE(docRoot string, client rpc.Client, datadir string, interactive bool) *jsre { - js := &jsre{ps1: "> "} - js.wait = make(chan *big.Int) - js.client = client - js.re = re.New(docRoot) - if err := js.apiBindings(); err != nil { - utils.Fatalf("Unable to initialize console - %v", err) - } - js.setupInput(datadir) - return js -} - -func newJSRE(stack *node.Node, docRoot, corsDomain string, client rpc.Client, interactive bool) *jsre { - js := &jsre{stack: stack, ps1: "> "} - // set default cors domain used by startRpc from CLI flag - js.corsDomain = corsDomain - js.wait = make(chan *big.Int) - js.client = client - js.re = re.New(docRoot) - if err := js.apiBindings(); err != nil { - utils.Fatalf("Unable to connect - %v", err) - } - js.setupInput(stack.DataDir()) - return js -} - -func (self *jsre) setupInput(datadir string) { - self.withHistory(datadir, func(hist *os.File) { utils.Stdin.ReadHistory(hist) }) - utils.Stdin.SetCtrlCAborts(true) - utils.Stdin.SetWordCompleter(makeCompleter(self)) - utils.Stdin.SetTabCompletionStyle(liner.TabPrints) - self.atexit = func() { - self.withHistory(datadir, func(hist *os.File) { - hist.Truncate(0) - utils.Stdin.WriteHistory(hist) - }) - utils.Stdin.Close() - close(self.wait) - } -} - -func (self *jsre) batch(statement string) { - err := self.re.EvalAndPrettyPrint(statement) - - if err != nil { - fmt.Printf("%v", jsErrorString(err)) - } - - if self.atexit != nil { - self.atexit() - } - - self.re.Stop(false) -} - -// show summary of current geth instance -func (self *jsre) welcome() { - self.re.Run(` - (function () { - console.log('instance: ' + web3.version.node); - console.log("coinbase: " + eth.coinbase); - var ts = 1000 * eth.getBlock(eth.blockNumber).timestamp; - console.log("at block: " + eth.blockNumber + " (" + new Date(ts) + ")"); - console.log(' datadir: ' + admin.datadir); - })(); - `) - if modules, err := self.supportedApis(); err == nil { - loadedModules := make([]string, 0) - for api, version := range modules { - loadedModules = append(loadedModules, fmt.Sprintf("%s:%s", api, version)) - } - sort.Strings(loadedModules) - } -} - -func (self *jsre) supportedApis() (map[string]string, error) { - return self.client.SupportedModules() -} - -func (js *jsre) apiBindings() error { - apis, err := js.supportedApis() - if err != nil { - return err - } - - apiNames := make([]string, 0, len(apis)) - for a, _ := range apis { - apiNames = append(apiNames, a) - } - - jeth := utils.NewJeth(js.re, js.client) - js.re.Set("jeth", struct{}{}) - t, _ := js.re.Get("jeth") - jethObj := t.Object() - - jethObj.Set("send", jeth.Send) - jethObj.Set("sendAsync", jeth.Send) - - err = js.re.Compile("bignumber.js", re.BigNumber_JS) - if err != nil { - utils.Fatalf("Error loading bignumber.js: %v", err) - } - - err = js.re.Compile("web3.js", re.Web3_JS) - if err != nil { - utils.Fatalf("Error loading web3.js: %v", err) - } - - _, err = js.re.Run("var Web3 = require('web3');") - if err != nil { - utils.Fatalf("Error requiring web3: %v", err) - } - - _, err = js.re.Run("var web3 = new Web3(jeth);") - if err != nil { - utils.Fatalf("Error setting web3 provider: %v", err) - } - - // load only supported API's in javascript runtime - shortcuts := "var eth = web3.eth; var personal = web3.personal; " - for _, apiName := range apiNames { - if apiName == "web3" || apiName == "rpc" { - continue // manually mapped or ignore - } - - if jsFile, ok := web3ext.Modules[apiName]; ok { - if err = js.re.Compile(fmt.Sprintf("%s.js", apiName), jsFile); err == nil { - shortcuts += fmt.Sprintf("var %s = web3.%s; ", apiName, apiName) - } else { - utils.Fatalf("Error loading %s.js: %v", apiName, err) - } - } - } - - _, err = js.re.Run(shortcuts) - if err != nil { - utils.Fatalf("Error setting namespaces: %v", err) - } - - // overrule some of the methods that require password as input and ask for it interactively - p, err := js.re.Get("personal") - if err != nil { - fmt.Println("Unable to overrule sensitive methods in personal module") - return nil - } - - // Override the unlockAccount and newAccount methods on the personal object since these require user interaction. - // Assign the jeth.unlockAccount and jeth.newAccount in the jsre the original web3 callbacks. These will be called - // by the jeth.* methods after they got the password from the user and send the original web3 request to the backend. - if persObj := p.Object(); persObj != nil { // make sure the personal api is enabled over the interface - js.re.Run(`jeth.unlockAccount = personal.unlockAccount;`) - persObj.Set("unlockAccount", jeth.UnlockAccount) - js.re.Run(`jeth.newAccount = personal.newAccount;`) - persObj.Set("newAccount", jeth.NewAccount) - } - - // The admin.sleep and admin.sleepBlocks are offered by the console and not by the RPC layer. - // Bind these if the admin module is available. - if a, err := js.re.Get("admin"); err == nil { - if adminObj := a.Object(); adminObj != nil { - adminObj.Set("sleepBlocks", jeth.SleepBlocks) - adminObj.Set("sleep", jeth.Sleep) - } - } - - return nil -} - -func (self *jsre) AskPassword() (string, bool) { - pass, err := utils.Stdin.PasswordPrompt("Passphrase: ") - if err != nil { - return "", false - } - return pass, true -} - -func (self *jsre) ConfirmTransaction(tx string) bool { - // Retrieve the Ethereum instance from the node - var ethereum *eth.Ethereum - if err := self.stack.Service(ðereum); err != nil { - return false - } - // If natspec is enabled, ask for permission - if ethereum.NatSpec && false /* disabled for now */ { - // notice := natspec.GetNotice(self.xeth, tx, ethereum.HTTPClient()) - // fmt.Println(notice) - // answer, _ := self.Prompt("Confirm Transaction [y/n]") - // return strings.HasPrefix(strings.Trim(answer, " "), "y") - } - return true -} - -func (self *jsre) UnlockAccount(addr []byte) bool { - fmt.Printf("Please unlock account %x.\n", addr) - pass, err := utils.Stdin.PasswordPrompt("Passphrase: ") - if err != nil { - return false - } - // TODO: allow retry - var ethereum *eth.Ethereum - if err := self.stack.Service(ðereum); err != nil { - return false - } - a := accounts.Account{Address: common.BytesToAddress(addr)} - if err := ethereum.AccountManager().Unlock(a, pass); err != nil { - return false - } else { - fmt.Println("Account is now unlocked for this session.") - return true - } -} - -// preloadJSFiles loads JS files that the user has specified with ctx.PreLoadJSFlag into -// the JSRE. If not all files could be loaded it will return an error describing the error. -func (self *jsre) preloadJSFiles(ctx *cli.Context) error { - if ctx.GlobalString(utils.PreLoadJSFlag.Name) != "" { - assetPath := ctx.GlobalString(utils.JSpathFlag.Name) - jsFiles := strings.Split(ctx.GlobalString(utils.PreLoadJSFlag.Name), ",") - for _, file := range jsFiles { - filename := common.AbsolutePath(assetPath, strings.TrimSpace(file)) - if err := self.re.Exec(filename); err != nil { - return fmt.Errorf("%s: %v", file, jsErrorString(err)) - } - } - } - return nil -} - -// jsErrorString adds a backtrace to errors generated by otto. -func jsErrorString(err error) string { - if ottoErr, ok := err.(*otto.Error); ok { - return ottoErr.String() - } - return err.Error() -} - -func (self *jsre) interactive() { - // Read input lines. - prompt := make(chan string) - inputln := make(chan string) - go func() { - defer close(inputln) - for { - line, err := utils.Stdin.Prompt(<-prompt) - if err != nil { - if err == liner.ErrPromptAborted { // ctrl-C - self.resetPrompt() - inputln <- "" - continue - } - return - } - inputln <- line - } - }() - // Wait for Ctrl-C, too. - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - - defer func() { - if self.atexit != nil { - self.atexit() - } - self.re.Stop(false) - }() - for { - prompt <- self.ps1 - select { - case <-sig: - fmt.Println("caught interrupt, exiting") - return - case input, ok := <-inputln: - if !ok || indentCount <= 0 && exit.MatchString(input) { - return - } - if onlyws.MatchString(input) { - continue - } - str += input + "\n" - self.setIndent() - if indentCount <= 0 { - if mustLogInHistory(str) { - utils.Stdin.AppendHistory(str[:len(str)-1]) - } - self.parseInput(str) - str = "" - } - } - } -} - -func mustLogInHistory(input string) bool { - return len(input) == 0 || - passwordRegexp.MatchString(input) || - !leadingSpace.MatchString(input) -} - -func (self *jsre) withHistory(datadir string, op func(*os.File)) { - hist, err := os.OpenFile(filepath.Join(datadir, "history"), os.O_RDWR|os.O_CREATE, os.ModePerm) - if err != nil { - fmt.Printf("unable to open history file: %v\n", err) - return - } - op(hist) - hist.Close() -} - -func (self *jsre) parseInput(code string) { - defer func() { - if r := recover(); r != nil { - fmt.Println("[native] error", r) - } - }() - if err := self.re.EvalAndPrettyPrint(code); err != nil { - if ottoErr, ok := err.(*otto.Error); ok { - fmt.Println(ottoErr.String()) - } else { - fmt.Println(err) - } - return - } -} - -var indentCount = 0 -var str = "" - -func (self *jsre) resetPrompt() { - indentCount = 0 - str = "" - self.ps1 = "> " -} - -func (self *jsre) setIndent() { - open := strings.Count(str, "{") - open += strings.Count(str, "(") - closed := strings.Count(str, "}") - closed += strings.Count(str, ")") - indentCount = open - closed - if indentCount <= 0 { - self.ps1 = "> " - } else { - self.ps1 = strings.Join(make([]string, indentCount*2), "..") - self.ps1 += " " - } -} diff --git a/cmd/geth/js_test.go b/cmd/geth/js_test.go deleted file mode 100644 index 7459f69575..0000000000 --- a/cmd/geth/js_test.go +++ /dev/null @@ -1,502 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "fmt" - "io/ioutil" - "math/big" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/compiler" - "github.com/ethereum/go-ethereum/common/httpclient" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/node" -) - -const ( - testSolcPath = "" - solcVersion = "0.9.23" - testAddress = "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182" - testBalance = "10000000000000000000" - // of empty string - testHash = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" -) - -var ( - versionRE = regexp.MustCompile(strconv.Quote(`"compilerVersion":"` + solcVersion + `"`)) - testNodeKey, _ = crypto.HexToECDSA("4b50fa71f5c3eeb8fdc452224b2395af2fcc3d125e06c32c82e048c0559db03f") - testAccount, _ = crypto.HexToECDSA("e6fab74a43941f82d89cb7faa408e227cdad3153c4720e540e855c19b15e6674") - testGenesis = `{"` + testAddress[2:] + `": {"balance": "` + testBalance + `"}}` -) - -type testjethre struct { - *jsre - lastConfirm string - client *httpclient.HTTPClient -} - -// Temporary disabled while natspec hasn't been migrated -//func (self *testjethre) ConfirmTransaction(tx string) bool { -// var ethereum *eth.Ethereum -// self.stack.Service(ðereum) -// -// if ethereum.NatSpec { -// self.lastConfirm = natspec.GetNotice(self.xeth, tx, self.client) -// } -// return true -//} - -func testJEthRE(t *testing.T) (string, *testjethre, *node.Node) { - return testREPL(t, nil) -} - -func testREPL(t *testing.T, config func(*eth.Config)) (string, *testjethre, *node.Node) { - tmp, err := ioutil.TempDir("", "geth-test") - if err != nil { - t.Fatal(err) - } - // Create a networkless protocol stack - stack, err := node.New(&node.Config{DataDir: tmp, PrivateKey: testNodeKey, Name: "test", NoDiscovery: true}) - if err != nil { - t.Fatalf("failed to create node: %v", err) - } - // Initialize and register the Ethereum protocol - accman := accounts.NewPlaintextManager(filepath.Join(tmp, "keystore")) - db, _ := ethdb.NewMemDatabase() - core.WriteGenesisBlockForTesting(db, core.GenesisAccount{ - Address: common.HexToAddress(testAddress), - Balance: common.String2Big(testBalance), - }) - ethConf := ð.Config{ - ChainConfig: &core.ChainConfig{HomesteadBlock: new(big.Int)}, - TestGenesisState: db, - AccountManager: accman, - DocRoot: "/", - SolcPath: testSolcPath, - PowTest: true, - } - if config != nil { - config(ethConf) - } - if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { - return eth.New(ctx, ethConf) - }); err != nil { - t.Fatalf("failed to register ethereum protocol: %v", err) - } - // Initialize all the keys for testing - a, err := accman.ImportECDSA(testAccount, "") - if err != nil { - t.Fatal(err) - } - if err := accman.Unlock(a, ""); err != nil { - t.Fatal(err) - } - // Start the node and assemble the REPL tester - if err := stack.Start(); err != nil { - t.Fatalf("failed to start test stack: %v", err) - } - var ethereum *eth.Ethereum - stack.Service(ðereum) - - assetPath := filepath.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "cmd", "mist", "assets", "ext") - client, err := stack.Attach() - if err != nil { - t.Fatalf("failed to attach to node: %v", err) - } - tf := &testjethre{client: ethereum.HTTPClient()} - repl := newJSRE(stack, assetPath, "", client, false) - tf.jsre = repl - return tmp, tf, stack -} - -func TestNodeInfo(t *testing.T) { - t.Skip("broken after p2p update") - tmp, repl, ethereum := testJEthRE(t) - defer ethereum.Stop() - defer os.RemoveAll(tmp) - - want := `{"DiscPort":0,"IP":"0.0.0.0","ListenAddr":"","Name":"test","NodeID":"4cb2fc32924e94277bf94b5e4c983beedb2eabd5a0bc941db32202735c6625d020ca14a5963d1738af43b6ac0a711d61b1a06de931a499fe2aa0b1a132a902b5","NodeUrl":"enode://4cb2fc32924e94277bf94b5e4c983beedb2eabd5a0bc941db32202735c6625d020ca14a5963d1738af43b6ac0a711d61b1a06de931a499fe2aa0b1a132a902b5@0.0.0.0:0","TCPPort":0,"Td":"131072"}` - checkEvalJSON(t, repl, `admin.nodeInfo`, want) -} - -func TestAccounts(t *testing.T) { - tmp, repl, node := testJEthRE(t) - defer node.Stop() - defer os.RemoveAll(tmp) - - checkEvalJSON(t, repl, `eth.accounts`, `["`+testAddress+`"]`) - checkEvalJSON(t, repl, `eth.coinbase`, `"`+testAddress+`"`) - val, err := repl.re.Run(`jeth.newAccount("password")`) - if err != nil { - t.Errorf("expected no error, got %v", err) - } - addr := val.String() - if !regexp.MustCompile(`0x[0-9a-f]{40}`).MatchString(addr) { - t.Errorf("address not hex: %q", addr) - } - - checkEvalJSON(t, repl, `eth.accounts`, `["`+testAddress+`","`+addr+`"]`) - -} - -func TestBlockChain(t *testing.T) { - tmp, repl, node := testJEthRE(t) - defer node.Stop() - defer os.RemoveAll(tmp) - // get current block dump before export/import. - val, err := repl.re.Run("JSON.stringify(debug.dumpBlock(eth.blockNumber))") - if err != nil { - t.Errorf("expected no error, got %v", err) - } - beforeExport := val.String() - - // do the export - extmp, err := ioutil.TempDir("", "geth-test-export") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(extmp) - tmpfile := filepath.Join(extmp, "export.chain") - tmpfileq := strconv.Quote(tmpfile) - - var ethereum *eth.Ethereum - node.Service(ðereum) - ethereum.BlockChain().Reset() - - checkEvalJSON(t, repl, `admin.exportChain(`+tmpfileq+`)`, `true`) - if _, err := os.Stat(tmpfile); err != nil { - t.Fatal(err) - } - - // check import, verify that dumpBlock gives the same result. - checkEvalJSON(t, repl, `admin.importChain(`+tmpfileq+`)`, `true`) - checkEvalJSON(t, repl, `debug.dumpBlock(eth.blockNumber)`, beforeExport) -} - -func TestMining(t *testing.T) { - tmp, repl, node := testJEthRE(t) - defer node.Stop() - defer os.RemoveAll(tmp) - checkEvalJSON(t, repl, `eth.mining`, `false`) -} - -func TestRPC(t *testing.T) { - tmp, repl, node := testJEthRE(t) - defer node.Stop() - defer os.RemoveAll(tmp) - - checkEvalJSON(t, repl, `admin.startRPC("127.0.0.1", 5004, "*", "web3,eth,net")`, `true`) -} - -func TestCheckTestAccountBalance(t *testing.T) { - t.Skip() // i don't think it tests the correct behaviour here. it's actually testing - // internals which shouldn't be tested. This now fails because of a change in the core - // and i have no means to fix this, sorry - @obscuren - tmp, repl, node := testJEthRE(t) - defer node.Stop() - defer os.RemoveAll(tmp) - - repl.re.Run(`primary = "` + testAddress + `"`) - checkEvalJSON(t, repl, `eth.getBalance(primary)`, `"`+testBalance+`"`) -} - -func TestSignature(t *testing.T) { - tmp, repl, node := testJEthRE(t) - defer node.Stop() - defer os.RemoveAll(tmp) - - val, err := repl.re.Run(`eth.sign("` + testAddress + `", "` + testHash + `")`) - - // This is a very preliminary test, lacking actual signature verification - if err != nil { - t.Errorf("Error running js: %v", err) - return - } - output := val.String() - t.Logf("Output: %v", output) - - regex := regexp.MustCompile(`^0x[0-9a-f]{130}$`) - if !regex.MatchString(output) { - t.Errorf("Signature is not 65 bytes represented in hexadecimal.") - return - } -} - -func TestContract(t *testing.T) { - t.Skip("contract testing is implemented with mining in ethash test mode. This takes about 7seconds to run. Unskip and run on demand") - coinbase := common.HexToAddress(testAddress) - tmp, repl, ethereum := testREPL(t, func(conf *eth.Config) { - conf.Etherbase = coinbase - conf.PowTest = true - }) - if err := ethereum.Start(); err != nil { - t.Errorf("error starting ethereum: %v", err) - return - } - defer ethereum.Stop() - defer os.RemoveAll(tmp) - - // Temporary disabled while registrar isn't migrated - //reg := registrar.New(repl.xeth) - //_, err := reg.SetGlobalRegistrar("", coinbase) - //if err != nil { - // t.Errorf("error setting HashReg: %v", err) - //} - //_, err = reg.SetHashReg("", coinbase) - //if err != nil { - // t.Errorf("error setting HashReg: %v", err) - //} - //_, err = reg.SetUrlHint("", coinbase) - //if err != nil { - // t.Errorf("error setting HashReg: %v", err) - //} - /* TODO: - * lookup receipt and contract addresses by tx hash - * name registration for HashReg and UrlHint addresses - * mine those transactions - * then set once more SetHashReg SetUrlHint - */ - - source := `contract test {\n` + - " /// @notice Will multiply `a` by 7." + `\n` + - ` function multiply(uint a) returns(uint d) {\n` + - ` return a * 7;\n` + - ` }\n` + - `}\n` - - if checkEvalJSON(t, repl, `admin.stopNatSpec()`, `true`) != nil { - return - } - - contractInfo, err := ioutil.ReadFile("info_test.json") - if err != nil { - t.Fatalf("%v", err) - } - if checkEvalJSON(t, repl, `primary = eth.accounts[0]`, `"`+testAddress+`"`) != nil { - return - } - if checkEvalJSON(t, repl, `source = "`+source+`"`, `"`+source+`"`) != nil { - return - } - - // if solc is found with right version, test it, otherwise read from file - sol, err := compiler.New("") - if err != nil { - t.Logf("solc not found: mocking contract compilation step") - } else if sol.Version() != solcVersion { - t.Logf("WARNING: solc different version found (%v, test written for %v, may need to update)", sol.Version(), solcVersion) - } - - if err != nil { - info, err := ioutil.ReadFile("info_test.json") - if err != nil { - t.Fatalf("%v", err) - } - _, err = repl.re.Run(`contract = JSON.parse(` + strconv.Quote(string(info)) + `)`) - if err != nil { - t.Errorf("%v", err) - } - } else { - if checkEvalJSON(t, repl, `contract = eth.compile.solidity(source).test`, string(contractInfo)) != nil { - return - } - } - - if checkEvalJSON(t, repl, `contract.code`, `"0x605880600c6000396000f3006000357c010000000000000000000000000000000000000000000000000000000090048063c6888fa114602e57005b603d6004803590602001506047565b8060005260206000f35b60006007820290506053565b91905056"`) != nil { - return - } - - if checkEvalJSON( - t, repl, - `contractaddress = eth.sendTransaction({from: primary, data: contract.code})`, - `"0x46d69d55c3c4b86a924a92c9fc4720bb7bce1d74"`, - ) != nil { - return - } - - if !processTxs(repl, t, 8) { - return - } - - callSetup := `abiDef = JSON.parse('[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}]'); -Multiply7 = eth.contract(abiDef); -multiply7 = Multiply7.at(contractaddress); -` - _, err = repl.re.Run(callSetup) - if err != nil { - t.Errorf("unexpected error setting up contract, got %v", err) - return - } - - expNotice := "" - if repl.lastConfirm != expNotice { - t.Errorf("incorrect confirmation message: expected %v, got %v", expNotice, repl.lastConfirm) - return - } - - if checkEvalJSON(t, repl, `admin.startNatSpec()`, `true`) != nil { - return - } - if checkEvalJSON(t, repl, `multiply7.multiply.sendTransaction(6, { from: primary })`, `"0x4ef9088431a8033e4580d00e4eb2487275e031ff4163c7529df0ef45af17857b"`) != nil { - return - } - - if !processTxs(repl, t, 1) { - return - } - - expNotice = `About to submit transaction (no NatSpec info found for contract: content hash not found for '0x87e2802265838c7f14bb69eecd2112911af6767907a702eeaa445239fb20711b'): {"params":[{"to":"0x46d69d55c3c4b86a924a92c9fc4720bb7bce1d74","data": "0xc6888fa10000000000000000000000000000000000000000000000000000000000000006"}]}` - if repl.lastConfirm != expNotice { - t.Errorf("incorrect confirmation message: expected\n%v, got\n%v", expNotice, repl.lastConfirm) - return - } - - var contentHash = `"0x86d2b7cf1e72e9a7a3f8d96601f0151742a2f780f1526414304fbe413dc7f9bd"` - if sol != nil && solcVersion != sol.Version() { - modContractInfo := versionRE.ReplaceAll(contractInfo, []byte(`"compilerVersion":"`+sol.Version()+`"`)) - fmt.Printf("modified contractinfo:\n%s\n", modContractInfo) - contentHash = `"` + common.ToHex(crypto.Keccak256([]byte(modContractInfo))) + `"` - } - if checkEvalJSON(t, repl, `filename = "/tmp/info.json"`, `"/tmp/info.json"`) != nil { - return - } - if checkEvalJSON(t, repl, `contentHash = admin.saveInfo(contract.info, filename)`, contentHash) != nil { - return - } - if checkEvalJSON(t, repl, `admin.register(primary, contractaddress, contentHash)`, `true`) != nil { - return - } - if checkEvalJSON(t, repl, `admin.registerUrl(primary, contentHash, "file://"+filename)`, `true`) != nil { - return - } - - if checkEvalJSON(t, repl, `admin.startNatSpec()`, `true`) != nil { - return - } - - if !processTxs(repl, t, 3) { - return - } - - if checkEvalJSON(t, repl, `multiply7.multiply.sendTransaction(6, { from: primary })`, `"0x60 { - t.Errorf("%d trasactions were not mined", txc) - return f6d7635c12ad0b231e66da2f987ca3dfdca58ffe49c6442aa55960858103fd0c"`) != nil { - return - } - - if !processTxs(repl, t, 1) { - return - } - - expNotice = "Will multiply 6 by 7." - if repl.lastConfirm != expNotice { - t.Errorf("incorrect confirmation message: expected\n%v, got\n%v", expNotice, repl.lastConfirm) - return - } -} - -func pendingTransactions(repl *testjethre, t *testing.T) (txc int64, err error) { - var ethereum *eth.Ethereum - repl.stack.Service(ðereum) - - txs := ethereum.TxPool().GetTransactions() - return int64(len(txs)), nil -} - -func processTxs(repl *testjethre, t *testing.T, expTxc int) bool { - var txc int64 - var err error - for i := 0; i < 50; i++ { - txc, err = pendingTransactions(repl, t) - if err != nil { - t.Errorf("unexpected error checking pending transactions: %v", err) - return false - } - if expTxc < int(txc) { - t.Errorf("too many pending transactions: expected %v, got %v", expTxc, txc) - return false - } else if expTxc == int(txc) { - break - } - time.Sleep(100 * time.Millisecond) - } - if int(txc) != expTxc { - t.Errorf("incorrect number of pending transactions, expected %v, got %v", expTxc, txc) - return false - } - var ethereum *eth.Ethereum - repl.stack.Service(ðereum) - - err = ethereum.StartMining(runtime.NumCPU(), "") - if err != nil { - t.Errorf("unexpected error mining: %v", err) - return false - } - defer ethereum.StopMining() - - timer := time.NewTimer(100 * time.Second) - blockNr := ethereum.BlockChain().CurrentBlock().Number() - height := new(big.Int).Add(blockNr, big.NewInt(1)) - repl.wait <- height - select { - case <-timer.C: - // if times out make sure the xeth loop does not block - go func() { - select { - case repl.wait <- nil: - case <-repl.wait: - } - }() - case <-repl.wait: - } - txc, err = pendingTransactions(repl, t) - if err != nil { - t.Errorf("unexpected error checking pending transactions: %v", err) - return false - } - if txc != 0 { - t.Errorf("%d trasactions were not mined", txc) - return false - } - return true -} - -func checkEvalJSON(t *testing.T, re *testjethre, expr, want string) error { - val, err := re.re.Run("JSON.stringify(" + expr + ")") - if err == nil && val.String() != want { - err = fmt.Errorf("Output mismatch for `%s`:\ngot: %s\nwant: %s", expr, val.String(), want) - } - if err != nil { - _, file, line, _ := runtime.Caller(1) - file = filepath.Base(file) - fmt.Printf("\t%s:%d: %v\n", file, line, err) - t.Fail() - } - return err -} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 3ea47fc297..bedc1254a1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -29,7 +29,7 @@ import ( "time" "github.com/ethereum/ethash" - "github.com/ethereum/go-ethereum/accounts" + // "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" @@ -322,18 +322,18 @@ func startNode(ctx *cli.Context, stack *node.Node) { utils.StartNode(stack) // Unlock any account specifically requested - var accman *accounts.Manager - if err := stack.Service(&accman); err != nil { - utils.Fatalf("ethereum service not running: %v", err) - } - passwords := utils.MakePasswordList(ctx) + // var accman *accounts.Manager + // if err := stack.Service(&accman); err != nil { + // utils.Fatalf("ethereum service not running: %v", err) + // } + // passwords := utils.MakePasswordList(ctx) - accounts := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",") - for i, account := range accounts { - if trimmed := strings.TrimSpace(account); trimmed != "" { - unlockAccount(ctx, accman, trimmed, i, passwords) - } - } + // accounts := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",") + // for i, account := range accounts { + // if trimmed := strings.TrimSpace(account); trimmed != "" { + // unlockAccount(ctx, accman, trimmed, i, passwords) + // } + // } // Start auxiliary services if enabled if ctx.GlobalBool(utils.MiningEnabledFlag.Name) { var ethereum *eth.Ethereum diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 3083ea4e8b..3b521a0e12 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -24,7 +24,6 @@ import ( "os/signal" "regexp" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -210,57 +209,3 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las glog.Infoln("Exported blockchain to ", fn) return nil } - -// tries unlocking the specified account a few times. -func UnlockAccount(accman *accounts.Manager, address string, i int, passwords []string) (accounts.Account, string) { - account, err := MakeAddress(accman, address) - if err != nil { - Fatalf("Could not list accounts: %v", err) - } - for trials := 0; trials < 3; trials++ { - prompt := fmt.Sprintf("Unlocking account %s | Attempt %d/%d", address, trials+1, 3) - password := GetPassPhrase(prompt, false, i, passwords) - err = accman.Unlock(account, password) - if err == nil { - glog.V(logger.Info).Infof("Unlocked account %x", account.Address) - return account, password - } - if err, ok := err.(*accounts.AmbiguousAddrError); ok { - glog.V(logger.Info).Infof("Unlocked account %x", account.Address) - return ambiguousAddrRecovery(accman, err, password), password - } - if err != accounts.ErrDecrypt { - // No need to prompt again if the error is not decryption-related. - break - } - } - // All trials expended to unlock account, bail out - Fatalf("Failed to unlock account %s (%v)", address, err) - return accounts.Account{}, "" -} - -func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrError, auth string) accounts.Account { - fmt.Printf("Multiple key files exist for address %x:\n", err.Addr) - for _, a := range err.Matches { - fmt.Println(" ", a.File) - } - fmt.Println("Testing your passphrase against all of them...") - var match *accounts.Account - for _, a := range err.Matches { - if err := am.Unlock(a, auth); err == nil { - match = &a - break - } - } - if match == nil { - Fatalf("None of the listed files could be unlocked.") - } - fmt.Printf("Your passphrase unlocked %s\n", match.File) - fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") - for _, a := range err.Matches { - if a != *match { - fmt.Println(" ", a.File) - } - } - return *match -} diff --git a/cmd/utils/customflags.go b/cmd/utils/customflags.go index 2714641104..5cbccfe98b 100644 --- a/cmd/utils/customflags.go +++ b/cmd/utils/customflags.go @@ -39,7 +39,7 @@ func (self *DirectoryString) String() string { } func (self *DirectoryString) Set(value string) error { - self.Value = ExpandPath(value) + self.Value = expandPath(value) return nil } @@ -135,7 +135,7 @@ func (self *DirectoryFlag) Set(value string) { // 2. expands embedded environment variables // 3. cleans the path, e.g. /a/b/../c -> /a/c // Note, it has limitations, e.g. ~someuser/tmp will not be expanded -func ExpandPath(p string) string { +func expandPath(p string) string { if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { if user, err := user.Current(); err == nil { p = user.HomeDir + p[1:] diff --git a/cmd/utils/customflags_test.go b/cmd/utils/customflags_test.go index 05e47abe4d..de39ca36a1 100644 --- a/cmd/utils/customflags_test.go +++ b/cmd/utils/customflags_test.go @@ -33,7 +33,7 @@ func TestPathExpansion(t *testing.T) { } os.Setenv("DDDXXX", "/tmp") for test, expected := range tests { - got := ExpandPath(test) + got := expandPath(test) if got != expected { t.Errorf("test %s, got %s, expected %s\n", test, got, expected) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 280e7241ca..b789e96c83 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -712,7 +712,7 @@ func MakeSystemNode(name, version string, relconf release.Config, extra []byte, accounts := strings.Split(ctx.GlobalString(UnlockedAccountFlag.Name), ",") for i, account := range accounts { if trimmed := strings.TrimSpace(account); trimmed != "" { - UnlockAccount(accman, trimmed, i, passwords) + UnlockAccount(ctx, accman, trimmed, i, passwords) } } diff --git a/cmd/utils/input.go b/cmd/utils/input.go deleted file mode 100644 index 21ed8b68de..0000000000 --- a/cmd/utils/input.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package utils - -import ( - "fmt" - "strings" - - "github.com/peterh/liner" -) - -// Holds the stdin line reader. -// Only this reader may be used for input because it keeps -// an internal buffer. -var Stdin = newUserInputReader() - -type userInputReader struct { - *liner.State - warned bool - supported bool - normalMode liner.ModeApplier - rawMode liner.ModeApplier -} - -func newUserInputReader() *userInputReader { - r := new(userInputReader) - // Get the original mode before calling NewLiner. - // This is usually regular "cooked" mode where characters echo. - normalMode, _ := liner.TerminalMode() - // Turn on liner. It switches to raw mode. - r.State = liner.NewLiner() - rawMode, err := liner.TerminalMode() - if err != nil || !liner.TerminalSupported() { - r.supported = false - } else { - r.supported = true - r.normalMode = normalMode - r.rawMode = rawMode - // Switch back to normal mode while we're not prompting. - normalMode.ApplyMode() - } - return r -} - -func (r *userInputReader) Prompt(prompt string) (string, error) { - if r.supported { - r.rawMode.ApplyMode() - defer r.normalMode.ApplyMode() - } else { - // liner tries to be smart about printing the prompt - // and doesn't print anything if input is redirected. - // Un-smart it by printing the prompt always. - fmt.Print(prompt) - prompt = "" - defer fmt.Println() - } - return r.State.Prompt(prompt) -} - -func (r *userInputReader) PasswordPrompt(prompt string) (passwd string, err error) { - if r.supported { - r.rawMode.ApplyMode() - defer r.normalMode.ApplyMode() - return r.State.PasswordPrompt(prompt) - } - if !r.warned { - fmt.Println("!! Unsupported terminal, password will be echoed.") - r.warned = true - } - // Just as in Prompt, handle printing the prompt here instead of relying on liner. - fmt.Print(prompt) - passwd, err = r.State.Prompt("") - fmt.Println() - return passwd, err -} - -func (r *userInputReader) ConfirmPrompt(prompt string) (bool, error) { - prompt = prompt + " [y/N] " - input, err := r.Prompt(prompt) - if len(input) > 0 && strings.ToUpper(input[:1]) == "Y" { - return true, nil - } - return false, err -} - -// getPassPhrase retrieves the password associated with an account, either fetched -// from a list of preloaded passphrases, or requested interactively from the user. -func GetPassPhrase(prompt string, confirmation bool, i int, passwords []string) string { - // If a list of passwords was supplied, retrieve from them - if len(passwords) > 0 { - if i < len(passwords) { - return passwords[i] - } - return passwords[len(passwords)-1] - } - // Otherwise prompt the user for the password - if prompt != "" { - fmt.Println(prompt) - } - password, err := Stdin.PasswordPrompt("Passphrase: ") - if err != nil { - Fatalf("Failed to read passphrase: %v", err) - } - if confirmation { - confirm, err := Stdin.PasswordPrompt("Repeat passphrase: ") - if err != nil { - Fatalf("Failed to read passphrase confirmation: %v", err) - } - if password != confirm { - Fatalf("Passphrases do not match") - } - } - return password -} diff --git a/eth/backend.go b/eth/backend.go index 8b02d129b6..6da44e4959 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -341,7 +341,7 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "debug", Version: "1.0", - Service: NewPrivateDebugAPI(s.ChainConfig(), s), + Service: NewPrivateDebugAPI(s.chainConfig, s), }, { Namespace: "net", Version: "1.0", @@ -379,8 +379,6 @@ func (s *Ethereum) Miner() *miner.Miner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } -func (s *Ethereum) GPO() *GasPriceOracle { return s.gpo } -func (s *Ethereum) ChainConfig() *core.ChainConfig { return s.chainConfig } func (s *Ethereum) TxPool() *core.TxPool { return s.txPool } func (s *Ethereum) EventMux() *event.TypeMux { return s.eventMux } func (s *Ethereum) Pow() *ethash.Ethash { return s.pow } diff --git a/eth/bind.go b/eth/bind.go index 95868f45af..9c430f072e 100644 --- a/eth/bind.go +++ b/eth/bind.go @@ -20,8 +20,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rlp" @@ -70,7 +68,7 @@ func (b *ContractBackend) HasCode(ctx context.Context, contract common.Address, // call with the specified data as the input. The pending flag requests execution // against the pending block, not the stable head of the chain. func (b *ContractBackend) ContractCall(ctx context.Context, contract common.Address, data []byte, pending bool) ([]byte, error) { - if ctx ==-- nil { + if ctx == nil { ctx = context.Background() } // Convert the input args to the API spec @@ -82,7 +80,7 @@ func (b *ContractBackend) ContractCall(ctx context.Context, contract common.Addr if pending { block = rpc.PendingBlockNumber } - // Execute the call and convert the output ba--ck to Go types + // Execute the call and convert the output back to Go types out, err := b.bcapi.Call(ctx, args, block) return common.FromHex(out), err } @@ -135,19 +133,14 @@ func (b *ContractBackend) SendTransaction(ctx context.Context, tx *types.Transac return err } -func (b *ContractBackend) GetTxReceipt(txhash common.Hash) *types.Receipt { - return core.GetReceipt(b.eth.ChainDb(), txhash) +func (b *ContractBackend) GetTxReceipt(txhash common.Hash) (map[string]interface{}, error) { + return b.txapi.GetTransactionReceipt(txhash) } -func (b *ContractBackend) BalanceAt(address common.Address) *big.Int { - currentState, err := state.New(b.eth.BlockChain().CurrentBlock().Root(), b.eth.ChainDb()) - if err != nil { - return new(big.Int) - } - return currentState.GetBalance(address) +func (b *ContractBackend) BalanceAt(address common.Address) (*big.Int, error) { + return b.bcapi.GetBalance(context.Background(), address, rpc.LatestBlockNumber) } -func (b *ContractBackend) CodeAt(address common.Address) string { - currentState, _ := state.New(b.eth.BlockChain().CurrentBlock().Root(), b.eth.ChainDb()) - return common.ToHex(currentState.GetCode(address)) +func (b *ContractBackend) CodeAt(address common.Address) (string, error) { + return b.bcapi.GetCode(context.Background(), address, rpc.LatestBlockNumber) } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 5d6dcc39d7..5ef1d59989 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -27,7 +27,6 @@ var Modules = map[string]string{ "rpc": RPC_JS, "shh": Shh_JS, "txpool": TxPool_JS, - "net": Net_JS, "bzz": Bzz_JS, "ens": ENS_JS, "chequebook": Chequebook_JS, @@ -123,8 +122,13 @@ web3._extend({ params: 2, inputFormatter: [null, null] }), - new web3._extend.Method( -{ + new web3._extend.Method({ + name: 'setContentHash', + call: 'ens_setContentHash', + params: 2, + inputFormatter: [null, null] + }), + new web3._extend.Method({ name: 'resolve', call: 'ens_resolve', params: 1, @@ -139,23 +143,13 @@ web3._extend({ property: 'chequebook', methods: [ - new web3._extend.Method( -{ + new web3._extend.Method({ name: 'deposit', call: 'chequebook_deposit', - } - out, err := b.txapi.GetTransactionCount(ctx, account, rpc.PendingBlockNumber) - return out.Uint64(), err -} - -// SuggestGasPrice implements bind.ContractTransactor retrieving the currently -// suggested gas price to allow a timely execution of a transaction. -func (b *ContractBackend params: 1, inputFormatter: [null] }), - new web3._extend.Propert -y({ + new web3._extend.Property({ name: 'balance', getter: 'chequebook_balance', outputFormatter: web3._extend.utils.toDecimal @@ -168,7 +162,6 @@ y({ }), new web3._extend.Method({ name: 'issue', - call: 'chequebook_issue', params: 2, inputFormatter: [null, null] @@ -177,50 +170,6 @@ y({ }); ` -const Personal_JS = ` - -web3._extend({ - property: 'personal', - methods: - [ - new web3._extend.Method({ - name: 'importRawKey', - call: 'personal_importRawKey', - params: 2 - }) - - ] -}); -` - -const TxPool_JS = `web3._extend({ - property: 'txpool', - methods: - [ - ], - properties: - [ - new web3._extend.Property({ - name: 'content', - getter: 'txpool_content' - }), - new web3._extend.Property({ - name: 'inspect', - getter: 'txpool_inspect' - }), - new web3._extend.Property({ - name: 'status', - getter: 'txpool_status', - outputFormatter: function(status) { - status.pending = web3._extend.utils.toDecimal(status.pending); - status.queued = web3._extend.utils.toDecimal(status.queued); - return status; - } - }) - ] -}); -`s - const Admin_JS = ` web3._extend({ property: 'admin', @@ -326,19 +275,6 @@ web3._extend({ }); ` -const Net_JS = ` -web3._extend({ - property: 'net', - methods: [], - properties: - [ - new web3._extend.Property({ - name: 'version', - getter: 'net_version' - }) - ] -}); -` const Debug_JS = ` web3._extend({ property: 'debug', diff --git a/swarm/api/api.go b/swarm/api/api.go index 883f226cf2..321837bda8 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -43,12 +43,12 @@ func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) { } // DPA reader API -func (self *Api) Retrieve(key storage.Key) storage.SectionReader { +func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { return self.dpa.Retrieve(key) } -func (self *Api) Store(data storage.SectionReader, wg *sync.WaitGroup) (key storage.Key, err error) { - return self.dpa.Store(data, wg) +func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) { + return self.dpa.Store(data, size, wg) } type ErrResolve error @@ -105,15 +105,15 @@ func (self *Api) parseAndResolve(uri string, nameresolver bool) (contentHash sto // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (string, error) { - sr := io.NewSectionReader(strings.NewReader(content), 0, int64(len(content))) + r := strings.NewReader(content) wg := &sync.WaitGroup{} - key, err := self.dpa.Store(sr, wg) + key, err := self.dpa.Store(r, int64(len(content)), wg) if err != nil { return "", err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) - sr = io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))) - key, err = self.dpa.Store(sr, wg) + r = strings.NewReader(manifest) + key, err = self.dpa.Store(r, int64(len(manifest)), wg) if err != nil { return "", err } @@ -124,11 +124,11 @@ func (self *Api) Put(content, contentType string) (string, error) { // Get uses iterative manifest retrieval and prefix matching // to resolve path to content using dpa retrieve // it returns a section reader, mimeType, status and an error -func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReader, mimeType string, status int, err error) { +func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) { key, _, path, err := self.parseAndResolve(uri, nameresolver) - - trie, err := loadManifest(self.dpa, key) + quitC := make(chan bool) + trie, err := loadManifest(self.dpa, key, quitC) if err != nil { glog.V(logger.Warn).Infof("[BZZ] Swarm: loadManifestTrie error: %v", err) return @@ -151,7 +151,8 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReade func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) { root, _, path, err := self.parseAndResolve(uri, nameresolver) - trie, err := loadManifest(self.dpa, root) + quitC := make(chan bool) + trie, err := loadManifest(self.dpa, root, quitC) if err != nil { return } @@ -162,9 +163,9 @@ func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) Hash: contentHash, ContentType: contentType, } - trie.addEntry(entry) + trie.addEntry(entry, quitC) } else { - trie.deleteEntry(path) + trie.deleteEntry(path, quitC) } err = trie.recalcAndStore() diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 0ee3ca8df7..0fb50e5793 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -1,11 +1,13 @@ package api import ( - // "bytes" + "io" "io/ioutil" "os" "testing" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -27,7 +29,7 @@ func testApi(t *testing.T, f func(*Api)) { } type testResponse struct { - reader storage.SectionReader + reader storage.LazySectionReader *Response } @@ -51,13 +53,14 @@ func checkResponse(t *testing.T, resp *testResponse, exp *Response) { resp.Content = string(content) } if resp.Content != exp.Content { - // if !bytes.Equal(resp.Content, exp.Content) { + // if !bytes.Equal(resp.Content, exp.Content) t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content)) } } // func expResponse(content []byte, mimeType string, status int) *Response { func expResponse(content string, mimeType string, status int) *Response { + glog.V(logger.Detail).Infof("expected content (%v): %v ", len(content), content) return &Response{mimeType, status, int64(len(content)), content} } @@ -67,7 +70,19 @@ func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { if err != nil { t.Fatalf("unexpected error: %v", err) } - return &testResponse{reader, &Response{mimeType, status, reader.Size(), ""}} + quitC := make(chan bool) + size, err := reader.Size(quitC) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + glog.V(logger.Detail).Infof("reader size: %v ", size) + s := make([]byte, size) + _, err = reader.Read(s) + if err != io.EOF { + t.Fatalf("unexpected error: %v", err) + } + reader.Seek(0, 0) + return &testResponse{reader, &Response{mimeType, status, size, string(s)}} // return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}} } diff --git a/swarm/api/config.go b/swarm/api/config.go index 6657167984..e9d4c87a8e 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -32,6 +32,7 @@ type Config struct { Port string PublicKey string BzzKey string + EnsRoot common.Address } // config is agnostic to where private key is coming from diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 9bed72d23d..25927b25b9 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -19,16 +19,15 @@ var ( "CacheCapacity": 5000, "Radius": 0, "Branches": 128, - "Hash": "SHA256", - "JoinTimeout": 120, - "SplitTimeout": 120, - "CallInterval": 10000000000, + "Hash": "SHA3", + "CallInterval": 3000000000, "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `", - "MaxProx": 10, - "ProxBinSize": 8, - "BucketSize": 3, + "MaxProx": 8, + "ProxBinSize": 2, + "BucketSize": 4, "PurgeInterval": 151200000000000, - "InitialRetryInterval": 4200000000, + "InitialRetryInterval": 42000000, + "MaxIdleInterval": 4200000000, "ConnRetryExp": 2, "Swap": { "BuyAt": 20000000000, @@ -67,7 +66,8 @@ var ( "Path": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e") + `", "Port": "8500", "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", - "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81" + "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", + "EnsRoot": "0x0000000000000000000000000000000000000000" }` ) diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 002bb95273..ac09602bc6 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -86,27 +86,29 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { errors := make([]error, cnt) done := make(chan bool, maxParallelFiles) dcnt := 0 + awg := &sync.WaitGroup{} for i, entry := range list { if i >= dcnt+maxParallelFiles { <-done dcnt++ } + awg.Add(1) go func(i int, entry *manifestTrieEntry, done chan bool) { f, err := os.Open(entry.Path) if err == nil { stat, _ := f.Stat() - sr := io.NewSectionReader(f, 0, stat.Size()) - wg := &sync.WaitGroup{} var hash storage.Key - hash, err = self.api.dpa.Store(sr, wg) + wg := &sync.WaitGroup{} + hash, err = self.api.dpa.Store(f, stat.Size(), wg) if hash != nil { list[i].Hash = hash.String() } wg.Wait() + awg.Done() if err == nil { first512 := make([]byte, 512) - fread, _ := sr.ReadAt(first512, 0) + fread, _ := f.ReadAt(first512, 0) if fread > 0 { mimeType := http.DetectContentType(first512[:fread]) if filepath.Ext(entry.Path) == ".css" { @@ -129,6 +131,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { trie := &manifestTrie{ dpa: self.api.dpa, } + quitC := make(chan bool) for i, entry := range list { if errors[i] != nil { return "", errors[i] @@ -140,9 +143,9 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { Hash: entry.Hash, ContentType: entry.ContentType, } - trie.addEntry(ientry) + trie.addEntry(ientry, quitC) } - trie.addEntry(entry) + trie.addEntry(entry, quitC) } err2 := trie.recalcAndStore() @@ -150,6 +153,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if err2 == nil { hs = trie.hash.String() } + awg.Wait() return hs, err2 } @@ -170,11 +174,13 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { if err != nil { return err } - // if len(path) > 0 { - // path += "/" - // } - trie, err := loadManifest(self.api.dpa, key) + if len(path) > 0 { + path += "/" + } + + quitC := make(chan bool) + trie, err := loadManifest(self.api.dpa, key, quitC) if err != nil { glog.V(logger.Warn).Infof("[BZZ] fs.Download: loadManifestTrie error: %v", err) return err @@ -186,72 +192,76 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { } var list []*downloadListEntry - var mde, mderr error + var mde error prevPath := lpath - err = trie.listWithPrefix(path, func(entry *manifestTrieEntry, suffix string) { // TODO: paralellize + err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) { glog.V(logger.Detail).Infof("[BZZ] fs.Download: %#v", entry) - key := common.Hex2Bytes(entry.Hash) + key = common.Hex2Bytes(entry.Hash) path := lpath + "/" + suffix dir := filepath.Dir(path) if dir != prevPath { mde = os.MkdirAll(dir, os.ModePerm) - if mde != nil { - mderr = mde - } prevPath = dir } if (mde == nil) && (path != dir+"/") { list = append(list, &downloadListEntry{key: key, path: path}) } }) - if err == nil { - err = mderr - } - - cnt := len(list) - errors := make([]error, cnt) - done := make(chan bool, maxParallelFiles) - dcnt := 0 - - for i, entry := range list { - if i >= dcnt+maxParallelFiles { - <-done - dcnt++ - } - go func(i int, entry *downloadListEntry, done chan bool) { - f, err := os.Create(entry.path) // TODO: path separators - if err == nil { - reader := self.api.dpa.Retrieve(entry.key) - writer := bufio.NewWriter(f) - _, err = io.CopyN(writer, reader, reader.Size()) // TODO: handle errors - err2 := writer.Flush() - if err == nil { - err = err2 - } - err2 = f.Close() - if err == nil { - err = err2 - } - } - - errors[i] = err - done <- true - }(i, entry, done) - } - for dcnt < cnt { - <-done - dcnt++ - } - if err != nil { return err } - for i, _ := range list { - if errors[i] != nil { - return errors[i] + + wg := sync.WaitGroup{} + errC := make(chan error) + done := make(chan bool, maxParallelFiles) + for i, entry := range list { + select { + case done <- true: + wg.Add(1) + case <-quitC: + return fmt.Errorf("aborted") } + go func(i int, entry *downloadListEntry) { + defer wg.Done() + f, err := os.Create(entry.path) // TODO: path separators + if err == nil { + + reader := self.api.dpa.Retrieve(entry.key) + writer := bufio.NewWriter(f) + size, err := reader.Size(quitC) + if err == nil { + _, err = io.CopyN(writer, reader, size) // TODO: handle errors + err2 := writer.Flush() + if err == nil { + err = err2 + } + err2 = f.Close() + if err == nil { + err = err2 + } + } + } + if err != nil { + select { + case errC <- err: + case <-quitC: + } + return + } + <-done + }(i, entry) } - return err + go func() { + wg.Wait() + close(errC) + }() + select { + case err = <-errC: + return err + case <-quitC: + return fmt.Errorf("aborted") + } + } diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index e0bb8915a9..f26c5df0ab 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -1,10 +1,12 @@ package api import ( + "bytes" "io/ioutil" "os" "path" "runtime" + "sync" "testing" ) @@ -26,9 +28,9 @@ func testFileSystem(t *testing.T, f func(*FileSystem)) { } func readPath(t *testing.T, parts ...string) string { - // func readPath(t *testing.T, parts ...string) []byte { file := path.Join(parts...) content, err := ioutil.ReadFile(file) + if err != nil { t.Fatalf("unexpected error reading '%v': %v", file, err) } @@ -36,14 +38,12 @@ func readPath(t *testing.T, parts ...string) string { } func TestApiDirUpload0(t *testing.T) { - // t.Skip("FIXME") testFileSystem(t, func(fs *FileSystem) { api := fs.api bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "") if err != nil { t.Fatalf("unexpected error: %v", err) } - content := readPath(t, testDir, "test0", "index.html") resp := testGet(t, api, bzzhash+"/index.html") exp := expResponse(content, "text/html; charset=utf-8", 0) @@ -54,17 +54,12 @@ func TestApiDirUpload0(t *testing.T) { exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - content = readPath(t, testDir, "test0", "img", "logo.png") - resp = testGet(t, api, bzzhash+"/img/logo.png") - exp = expResponse(content, "image/png", 0) - _, _, _, err = api.Get(bzzhash, true) if err == nil { t.Fatalf("expected error: %v", err) } downloadDir := path.Join(testDownloadDir, "test0") - os.RemoveAll(downloadDir) defer os.RemoveAll(downloadDir) err = fs.Download(bzzhash, downloadDir) if err != nil { @@ -77,12 +72,10 @@ func TestApiDirUpload0(t *testing.T) { if bzzhash != newbzzhash { t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash) } - }) } func TestApiDirUploadModify(t *testing.T) { - // t.Skip("FIXME") testFileSystem(t, func(fs *FileSystem) { api := fs.api bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "") @@ -96,12 +89,24 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true) + index, err := ioutil.ReadFile(path.Join(testDir, "test0", "index.html")) if err != nil { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true) + wg := &sync.WaitGroup{} + hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg) + wg.Wait() + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true) if err != nil { t.Errorf("unexpected error: %v", err) return diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go index 1bed107887..12e1c28966 100644 --- a/swarm/api/http/roundtripper_test.go +++ b/swarm/api/http/roundtripper_test.go @@ -10,6 +10,8 @@ import ( "github.com/ethereum/go-ethereum/common/httpclient" ) +const port = "3222" + func TestRoundTripper(t *testing.T) { serveMux := http.NewServeMux() serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -20,9 +22,9 @@ func TestRoundTripper(t *testing.T) { http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) } }) - go http.ListenAndServe(":8600", serveMux) + go http.ListenAndServe(":"+port, serveMux) - rt := &RoundTripper{Port: "8600"} + rt := &RoundTripper{Port: port} client := httpclient.New("/") client.RegisterProtocol("bzz", rt) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6ac2b113e6..f5810c5c0a 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -86,8 +86,10 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { return } } - raw = proto[1:5] == "bzzr" - nameresolver = proto[1:5] != "bzzi" + if len(proto) > 4 { + raw = proto[1:5] == "bzzr" + nameresolver = proto[1:5] != "bzzi" + } glog.V(logger.Debug).Infof( "[BZZ] Swarm: %s request over protocol %s '%s' received.", @@ -96,10 +98,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { switch { case r.Method == "POST" || r.Method == "PUT": - key, err := a.Store(io.NewSectionReader(&sequentialReader{ - reader: r.Body, - ahead: make(map[int64]chan bool), - }, 0, r.ContentLength), nil) + key, err := a.Store(r.Body, r.ContentLength, nil) if err == nil { glog.V(logger.Debug).Infof("[BZZ] Swarm: Content for %v stored", key.Log()) } else { @@ -165,7 +164,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { // retrieving content reader := a.Retrieve(key) - glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", reader.Size()) + quitC := make(chan bool) + size, err := reader.Size(quitC) + glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", size) // setting mime type qv := requestURL.Query() @@ -176,7 +177,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { w.Header().Set("Content-Type", mimeType) http.ServeContent(w, r, uri, forever(), reader) - glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType) + glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType) // retrieve path via manifest } else { @@ -203,7 +204,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { } else { status = 200 } - glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, reader.Size(), mimeType, status) + quitC := make(chan bool) + size, err := reader.Size(quitC) + glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status) http.ServeContent(w, r, path, forever(), reader) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 0337382ff0..a78d86f485 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" "sync" "github.com/ethereum/go-ethereum/common" @@ -35,24 +34,24 @@ type manifestTrieEntry struct { subtrie *manifestTrie } -func loadManifest(dpa *storage.DPA, hash storage.Key) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand glog.V(logger.Detail).Infof("[BZZ] manifest lookup key: '%v'.", hash.Log()) // retrieve manifest via DPA manifestReader := dpa.Retrieve(hash) - return readManifest(manifestReader, hash, dpa) + return readManifest(manifestReader, hash, dpa, quitC) } -func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *storage.DPA) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests - manifestData := make([]byte, manifestReader.Size()) - var size int - size, err = manifestReader.Read(manifestData) - if int64(size) < manifestReader.Size() { + size, err := manifestReader.Size(quitC) + manifestData := make([]byte, size) + read, err := manifestReader.Read(manifestData) + if int64(read) < size { glog.V(logger.Detail).Infof("[BZZ] Manifest %v not found.", hash.Log()) if err == nil { - err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", size, manifestReader.Size()) + err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size) } return } @@ -72,12 +71,12 @@ func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *s dpa: dpa, } for _, entry := range man.Entries { - trie.addEntry(entry) + trie.addEntry(entry, quitC) } return } -func (self *manifestTrie) addEntry(entry *manifestTrieEntry) { +func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { self.hash = nil // trie modified, hash needs to be re-calculated on demand if len(entry.Path) == 0 { @@ -98,11 +97,11 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) { } if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) { - if self.loadSubTrie(oldentry) != nil { + if self.loadSubTrie(oldentry, quitC) != nil { return } entry.Path = entry.Path[cpl:] - oldentry.subtrie.addEntry(entry) + oldentry.subtrie.addEntry(entry, quitC) oldentry.Hash = "" return } @@ -114,8 +113,8 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) { } entry.Path = entry.Path[cpl:] oldentry.Path = oldentry.Path[cpl:] - subtrie.addEntry(entry) - subtrie.addEntry(oldentry) + subtrie.addEntry(entry, quitC) + subtrie.addEntry(oldentry, quitC) self.entries[b] = &manifestTrieEntry{ Path: commonPrefix, @@ -135,7 +134,7 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { return } -func (self *manifestTrie) deleteEntry(path string) { +func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { self.hash = nil // trie modified, hash needs to be re-calculated on demand if len(path) == 0 { @@ -155,10 +154,10 @@ func (self *manifestTrie) deleteEntry(path string) { epl := len(entry.Path) if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { - if self.loadSubTrie(entry) != nil { + if self.loadSubTrie(entry, quitC) != nil { return } - entry.subtrie.deleteEntry(path[epl:]) + entry.subtrie.deleteEntry(path[epl:], quitC) entry.Hash = "" // remove subtree if it has less than 2 elements cnt, lastentry := entry.subtrie.getCountLast() @@ -198,24 +197,24 @@ func (self *manifestTrie) recalcAndStore() error { return err } - sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))) + sr := bytes.NewReader(manifest) wg := &sync.WaitGroup{} - key, err2 := self.dpa.Store(sr, wg) + key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg) wg.Wait() self.hash = key return err2 } -func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry) (err error) { +func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) { if entry.subtrie == nil { hash := common.Hex2Bytes(entry.Hash) - entry.subtrie, err = loadManifest(self.dpa, hash) + entry.subtrie, err = loadManifest(self.dpa, hash, quitC) entry.Hash = "" // might not match, should be recalculated } return } -func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *manifestTrieEntry, suffix string)) (err error) { +func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error { plen := len(prefix) var start, stop int if plen == 0 { @@ -227,6 +226,11 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma } for i := start; i <= stop; i++ { + select { + case <-quitC: + return fmt.Errorf("aborted") + default: + } entry := self.entries[i] if entry != nil { epl := len(entry.Path) @@ -236,11 +240,13 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma l = epl } if prefix[:l] == entry.Path[:l] { - sterr := self.loadSubTrie(entry) - if sterr == nil { - entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], cb) - } else { - err = sterr + err := self.loadSubTrie(entry, quitC) + if err != nil { + return err + } + err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb) + if err != nil { + return err } } } else { @@ -250,14 +256,14 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma } } } - return + return nil } -func (self *manifestTrie) listWithPrefix(prefix string, cb func(entry *manifestTrieEntry, suffix string)) (err error) { - return self.listWithPrefixInt(prefix, "", cb) +func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) { + return self.listWithPrefixInt(prefix, "", quitC, cb) } -func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, pos int) { +func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) { glog.V(logger.Detail).Infof("[BZZ] findPrefixOf(%s)", path) @@ -275,10 +281,10 @@ func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, p if (len(path) >= epl) && (path[:epl] == entry.Path) { glog.V(logger.Detail).Infof("[BZZ] entry.ContentType = %v", entry.ContentType) if entry.ContentType == manifestType { - if self.loadSubTrie(entry) != nil { + if self.loadSubTrie(entry, quitC) != nil { return nil, 0 } - entry, pos = entry.subtrie.findPrefixOf(path[epl:]) + entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC) if entry != nil { pos += epl } @@ -308,6 +314,7 @@ func RegularSlashes(path string) (res string) { func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) { path := RegularSlashes(spath) var pos int - entry, pos = self.findPrefixOf(path) + quitC := make(chan bool) + entry, pos = self.findPrefixOf(path, quitC) return entry, path[:pos] } diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 558bdfb51b..3143e5eaaf 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -10,18 +10,19 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -func manifest(paths ...string) (manifestReader storage.SectionReader) { +func manifest(paths ...string) (manifestReader storage.LazySectionReader) { var entries []string for _, path := range paths { entry := fmt.Sprintf(`{"path":"%s"}`, path) entries = append(entries, entry) } manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ",")) - return io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))) + return &storage.LazyTestSectionReader{io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))} } func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie { - trie, err := readManifest(manifest(paths...), nil, nil) + quitC := make(chan bool) + trie, err := readManifest(manifest(paths...), nil, nil, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 97b5590fdf..3dfc37f1c2 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -34,7 +34,11 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { if err != nil { return nil, err } - expsize := reader.Size() + quitC := make(chan bool) + expsize, err := reader.Size(quitC) + if err != nil { + return nil, err + } body := make([]byte, expsize) size, err := reader.Read(body) if int64(size) == expsize { @@ -43,6 +47,8 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { return &Response{mimeType, status, expsize, string(body[:size])}, err } +// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash, +// and merge on to it. creating an entry w conentType (mime) func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true) } diff --git a/swarm/cmd/README.md b/swarm/cmd/README.md new file mode 100644 index 0000000000..6be77fba37 --- /dev/null +++ b/swarm/cmd/README.md @@ -0,0 +1,229 @@ + +# install and setup swarm + +swarm is developed on a branch of the ethereum/go-ethereum repo +at this stage of the project there is no packages or binary distro, you need to a dev environment and compile from source. +[This document spells out a complete server setup on ubuntu linux](https://gist.github.com/zelig/74eb365752ceaacf15e860fb80eacb3e) including git/ssh/screen config, golang and compilation, node/npm and network monitoring (might contain a few bits that are tangential to swarm). + +Assuming you got your setup working, you will use the `swarm` command line tool to control your swarm of instances. +This command line tool is at the moment geared towards developers and testing. +It is likely that it will be replaced by two different tools, one for devel/testing and one for end users + + +The command can be used to update the code + +```shell +swarm update upstream/swarm +``` + +Then compile with + +```shell +godep go build -v ./cmd/geth +``` + +Make sure you have `GOPATH` variable set and also that the `swarm` executable is in your PATH. +These environment variables are relevant and set to the following defaults. +Make sure you are happy with them, otherwise change them, in which case best to put these lines in your `~/.profile`. + +``` +export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum +export GETH=$GETH_DIR/geth +export SWARM_DIR=~/bzz +export SWARM_NETWORK_ID=322 +``` + +* `GETH_DIR` points to your git working copy (given `GOPATH` its standardly under `$GOPATH/src/github.com/ethereum/go-ethereum`) +* `GETH` points to the `geth` executable compiled from the swarm branch. If you have systemwide install or use multiple geths you may need to change this, otherwise it is assumed you compile to the working copy of the repo. +* `SWARM_DIR` is the root directory for all swarm related stuff: logs, configs, as well as geth datadirs, make sure this dir is on a device with sufficient disk space +* `SWARM_NETWORK_ID`: this is by default the network id of the swarm testnet. If you run your own swarm, you need to change it, choose a number that is not likely chosen by others to avoid others joining you. + +# Deploying and remote control + +the swarm command supports remote update and remote control of your instances. +In our setting we assume you want to run a cluster of potentially remote swarm nodes each running a local cluster of instances +The only assumption is that you have (passwordless) ssh access set up to your swarm servers. +Assume `nodes.lst` is a list of nodes in the format of `username@ip` one per line. blank lines and lines commented out with `#` are ignored. + + +This copies the scripts found in `swarm/cmd/swarm` on all remote nodes listed in `nodes.lst` + +``` +swarm remote-update-scripts nodes.lst +``` + +If you just want to deploy a locally compiled binaries to all your remote nodes, this will fail if the remote instances are running, so make sure you stop them beforehand + +```shell +swarm remote-run nodes.lst swarm stop all +swarm remote-update-bin nodes.lst +``` + + + +Once you deployed the executables to the nodes, you can control them all with one command. For instance the following line initialises a cluster of two test swarm instances on each remote node. +Watch out, this will wipe your storage and all swarm related data + +```shell +swarm remote-run nodes.lst swarm init 2 +``` + + +To (re) start a particular instance on a specific remote node with alternative options (for instance mining and different logging verbosity), you can just: + +```shell +swarm remote-run cicada@3.3.0.1 'swarm restart 01 --mine --verbosity=0 --vmodule=swarm/*=5' +``` + +# Logging + +To check logs + +```shell +swarm log 00 # taillog flow +swarm remote-run cicada@3.3.0.1 swarm log 00 +``` + +You can view the log with a pager for an instance with + +``` +swarm viewlog 00 +``` + +Logs are preserved and viewable with the above commands even when nodes are offline +Each new run logs to a different file + +To purge logs + +``` +swarm cleanlog 01 +``` + +To remove all logs on all nodes: + +``` +swarm remote-run nodes.lst swarm cleanlog all +``` + +# upload and dowload + +upload and download via a running local instance + +```shell +swarm up 00 /path/to/file/or/directory +swarm down 01 hash /path/to/destination +``` + +upload via remote swarm proxy or public gateway + +```shell +swarm remote-up gateway-url /path/to/file/or/directory +wget -O- gateway-url/bzz:/swarm-url +``` + + +# Further examples + +```shell +# start with updaing +swarm update chambers + +# display CLI options given to geth used to launch swarm instance 02 +swarm options 02 + +# restart swarm instance 00 with alternatiev options +swarm restart 00 --mine --bzznosync --verbosity=0 --vmodule=swarm/*=6 + +# attach console to a running swarm instance +swarm attach 00 + +# execute a command; e.g., start mining on a running instance +swarm execute 00 'miner.stop(1)' + +# display static info about a instance (even if its offline) +swarm info 00 + +# displays the enode url of a running instance +swarm enode 01 + +# add peers to a running swarm instance +swarm addpeers 00 "enode://1033c1cada...@3.3.0.1:30301" + +# to compile a list of enodes from all instances on all remote nodes: +swarm remote-run nodes.lst 'swarm enode all' > enodes.lst + +# to add all peers to all instances on each node +for node in `cat nodes.lst|grep -v '^#'`; do scp enodes.lst $node:; done +swarm remote-run nodes.lst 'swarm addpeers enodes.lst' + +# if you run a local network and your nodes do not listen to external IPs +swarm remote-run pivot.lst 'swarm restart all' + +# to add just one or a few guardians and let the network bootstrap +# swarm remote-run 'swarm enode all' +swarm addpeers pivot.lst +# or directly +swarm addpeers all <(IP_ADDR='[::]' swarm enode 01|tr -d '"') + +# stop all running instances on the node +swarm stop all + +# stop all running instances on all remote nodes +swarm remote-run nodes.lst swarm stop all + +# display peer connection table of running instance 00 +swarm hive 00 + +# display peer connection table for a running instance and continually refresh every 4 seconds +swarm monitor 00 4 + +# display peer connection table for all running instance on a remote node and continually refresh every 10 seconds +swarm monitor cicada@3.3.0.1 all 10 + + +# configure eth-net-intelligence-api network monitoring client API for a node (the name argument appears as a prefix for all instances in your cluster) +swarm netstatconf cicada-sworm + +# restart the net monitor client API +swarm netstatun + +# configure eth-net-intelligence-api network monitoring client API and (re)start the monitor tool on all remote nodes +swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' + + +swarm remote nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' +``` + + +see also: + +* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/test +* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/cmd + +# ethereum netstats client setup + +## install + +nodejs and npm are prerequisites + +```shell +# MAC +brew install node npm +# ubuntu +sudo apt-get install npm nodejs +``` + +clone the git repo and install: + +``` +git clone git@github.com:cubedro/eth-net-intelligence-api.git +cd eth-net-intelligence-api +npm install +npm install -g pm2 +``` + +## configure and run netstats client for each node + +```shell +swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' +``` diff --git a/swarm/cmd/bzzhash/bzzhash.go b/swarm/cmd/bzzhash/bzzhash.go index d4df5fb50b..37df224bfd 100644 --- a/swarm/cmd/bzzhash/bzzhash.go +++ b/swarm/cmd/bzzhash/bzzhash.go @@ -3,7 +3,6 @@ package main import ( "fmt" - "io" "os" "runtime" @@ -24,15 +23,11 @@ func main() { } stat, _ := f.Stat() - sr := io.NewSectionReader(f, 0, stat.Size()) chunker := storage.NewTreeChunker(storage.NewChunkerParams()) - hash := make([]byte, chunker.KeySize()) - errC := chunker.Split(hash, sr, nil, nil) - err, ok := <-errC + key, err := chunker.Split(f, stat.Size(), nil, nil, nil) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) - } - if !ok { - fmt.Printf("%064x\n", hash) + } else { + fmt.Printf("%v\n", key) } } diff --git a/swarm/cmd/swarm/env.sh b/swarm/cmd/swarm/env.sh new file mode 100644 index 0000000000..8ceafcafc7 --- /dev/null +++ b/swarm/cmd/swarm/env.sh @@ -0,0 +1,6 @@ +export PATH=$HOME/bin:$PATH +export GETH_DIR=$HOME/bin +export SWARM_DIR= + +export NVM_DIR=$HOME/.nvm +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm \ No newline at end of file diff --git a/swarm/cmd/swarm/gethup.sh b/swarm/cmd/swarm/gethup.sh deleted file mode 100644 index e9b93f69bf..0000000000 --- a/swarm/cmd/swarm/gethup.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# Usage: -# bash /path/to/eth-utils/gethup.sh - -root=$1 # base directory to use for datadir and logs -shift -id=$1 # double digit instance id like 00 01 02 -shift -ip_addr=$1 # ip address to substitute -shift - -# logs are output to a date-tagged file for each run , while a link is -# created to the latest, so that monitoring be easier with the same filename -# TODO: use this if GETH not set -# GETH=geth -# echo "ls -l $GETH" -# ls -l $GETH - -# geth CLI params e.g., (dd=04, run=09) -datetag=`date "+%c%y%m%d-%H%M%S"|cut -d ' ' -f 5` -datadir=$root/data/$id # /tmp/eth/04 -log=$root/log/$id.$datetag.log # /tmp/eth/04.09.log -linklog=$root/log/$id.current.log # /tmp/eth/04.09.log -stablelog=$root/log/$id.log # /tmp/eth/04.09.log -password=$id # 04 -port=303$id # 34504 -bzzport=322$id # 32204 -rpcport=302$id # 3204 - -mkdir -p $root/data -mkdir -p $root/enodes -mkdir -p $root/pids -mkdir -p $root/log -ln -sf "$log" "$linklog" -# if we do not have an account, create one -# will not prompt for password, we use the double digit instance id as passwd -# NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN -keystoredir="$datadir/keystore/" -# echo "KeyStore dir: $keystoredir" -if [ ! -d "$keystoredir" ]; then - # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]" - # mkdir -p $datadir/keystore - $GETH --datadir $datadir --password <(echo -n $id) account new >/dev/null 2>&1 - # create account with password 00, 01, ... - # note that the account key will be stored also separately outside - # datadir - # this way you can safely clear the data directory and still keep your key - # under `/keystore/dd - # LS=`ls $datadir/keystore` - # echo $LS - while [ ! -d "$keystoredir" ]; do - echo "." - ((i++)) - if ((i>10)); then break; fi - sleep 1 - done - # echo "copying keys $datadir/keystore $root/keystore/$id" - mkdir -p $root/keystore/$id - cp -R "$datadir/keystore/" $root/keystore/$id -fi - -# # mkdir -p $datadir/keystore -# if [ ! -d "$datadir/keystore" ]; then -# echo "copying keys $root/keystore/$id $datadir/keystore" -# cp -R $root/keystore/$id/keystore/ $datadir/keystore/ -# fi - - -# query node's enode url -if [ $ip_addr="" ]; then - pattern='\d+\.\d+\.\d+\.\d+' - ip_addr="[::]" -else - pattern='\[\:\:\]' -fi - -geth="$GETH --datadir $datadir --port $port" - -# echo -n "enode for instance $id... " -if [ ! "$GETH" = "" ] && [ ! -f $root/enodes/$id.enode ]; then - cmd="$geth js <(echo 'console.log(admin.nodeInfo.enode); exit();') " - # echo $cmd '2>/dev/null |grep enode | perl -pe "s/'$pattern'/'$ip_addr'/g" | perl -pe "s/^/\"/; s/\s*$/\"/;" > '$root/enodes/$id.enode - eval $cmd 2>/dev/null |grep enode | perl -pe "s/$pattern/$ip_addr/g" | perl -pe "s/^/\"/; s/\s*\$/\"/;" > $root/enodes/$id.enode -fi -# cat $root/enodes/$id.enode -echo - -# copy cluster enodes list to node's static node list -# echo "copy cluster enodes list to node's static node list" -if [ -f $root/enodes.all ]; then - cp $root/enodes.all $datadir/static-nodes.json -fi - -if [ ! -f $root/pids/$id.pid ]; then - # bring up node `dd` (double digit) - # - using /
- # - listening on port 303dd, (like 30300, 30301, ...) - # - with the account unlocked - # - launching json-rpc server on port 81dd (like 8100, 8101, 8102, ...) - # echo "BZZKEY=$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'" - BZZKEY=`$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'` - echo -n "starting instance $id ($BZZKEY @ $datadir )..." - # echo "$geth \ - # --identity=$id \ - # --bzzaccount=$BZZKEY --bzzport=$bzzport \ - # --unlock=$BZZKEY \ - # --password=<(echo -n $id) \ - # --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \ - # 2>&1 | tee "$stablelog" > "$log" & # comment out if you pipe it to a tty etc. - # " >&2 - - if [ -f $log ] && [ -f $root/stablelog ]; then - cp $stablelog `cat $root/prevlog` - fi - echo $log > $root/prevlog - - $GETH --datadir=$datadir \ - --identity=$id \ - --bzzaccount=$BZZKEY --bzzport=$bzzport \ - --port=$port \ - --unlock=$BZZKEY \ - --password=<(echo -n $id) \ - --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \ - > "$stablelog" 2>&1 & # comment out if you pipe it to a tty etc. - - # wait until ready - # pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'` - # echo "pid: $pid" - # ps auxwww|grep geth|grep "ty=$id"|grep -v grep - # echo $pid > $root/pids/$id.pid - #echo $! > $root/pids/$id.pid - while true; do - $GETH --exec="net" attach ipc:$datadir/geth.ipc > /dev/null 2>&1 && break - sleep 1 - echo -n "." - if ((i++>10)); then - echo "instance $id failed to start" - exit 1 - fi - done - echo -n "started - " - pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'` - echo "pid: $pid" - # ps auxwww|grep geth|grep "ty=$id"|grep -v grep - echo $pid > $root/pids/$id.pid -fi - -# to bring up logs, uncomment -# tail -f $log diff --git a/swarm/cmd/swarm/swarm b/swarm/cmd/swarm/swarm new file mode 100755 index 0000000000..d9f5b099db --- /dev/null +++ b/swarm/cmd/swarm/swarm @@ -0,0 +1,759 @@ +#!/bin/bash +if [ "$GETH_DIR" = "" ]; then + if [ "$GOPATH" = "" ]; then echo "either GETH_DIR or GOPATH environment variable must be set"; exit 1; fi + export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum +fi + +if [ "$GETH" = "" ]; then + export GETH=$GETH_DIR/geth +fi + +if [ "$SWARM_NETWORK_ID" = "" ]; then export SWARM_NETWORK_ID=322; fi + +if [ "$SWARM_DIR" = "" ]; then export SWARM_DIR=$HOME/bzz; fi + +if [ "$IP_ADDR" = "" ]; then + # export IP_ADDR=`curl ipecho.net/plain 2>/dev/null;echo f` + export IP_ADDR= +fi + +root=$SWARM_DIR +network_id=$SWARM_NETWORK_ID +cmd=$1 +shift + +dir="$root/$network_id" + +tmpdir=/tmp + +function randomfile { + dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null +} + +# swarm attach 00 brings up a console attached to a running instance +function attach { + id=$1 + shift + echo "attaching console to instance $id" + cmd="$GETH --datadir=$dir/data/$id $* attach ipc:$dir/data/$id/geth.ipc" + # echo $cmd + eval $cmd + } + +# swarm attach 00 brings up a console attached to a running instance +function execute { + id=$1 + shift + # attach $id --exec "'$*' " + cmd="$GETH --datadir=$dir/data/$id --exec '$*' attach ipc:$dir/data/$id/geth.ipc" + # echo $cmd + eval $cmd +} + +# swarm hive 00 displays the kademlia table of the given running instance +function hive { + if [ "$1" = "all" ]; then + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i $log 2>&1 &" + + $GETH $opts --password=<(echo -n $id) > "$log" 2>&1 & # comment out if you pipe it to a tty etc. + ln -sf "$log" "$linklog" + + # wait until ready + + ((j=0)) + while true; do + execute $id "net" > /dev/null 2>&1 && break + sleep 1 + echo -n "." + if ((j++>10)); then + echo "instance $id failed to start" + exit 1 + fi + done + echo -n "started - " + pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'` + echo "pid: $pid" + echo $pid > $dir/pids/$id.pid +} + + +# setup 00 creates the direcories for instance +function setup { + id=$1 + shift + mkdir -p $dir/data/$id + mkdir -p $dir/enodes + mkdir -p $dir/pids + mkdir -p $dir/log +} + +# creates the swarm base account for an instance +function create-account { + id=$1 + datadir=$dir/data/$id + # if we do not have an account, create one + # will not prompt for password, we use the double digit instance id as passwd + # NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN + keystoredir="$datadir/keystore/" + # echo "KeyStore dir: $keystoredir" + if [ ! -d "$keystoredir" ]; then + # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]" + # mkdir -p $datadir/keystore + $GETH --datadir=$datadir --password=<(echo -n $id) account new >/dev/null 2>&1 + # create account with password 00, 01, ... + # note that the account key will be stored also separately outside + # datadir + # this way you can safely clear the data directory and still keep your key + # under /keystore/dd + # LS=$(ls $datadir/keystore) + # echo $LS + while [ ! -d "$keystoredir" ]; do + echo "." + ((i++)) + if ((i>10)); then break; fi + sleep 1 + done + # echo "copying keys $datadir/keystore $root/keystore/$id" + mkdir -p $dir/keystore/$id + cp -R "$keystoredir" $dir/keystore/$id + fi +} + +# shuts down a running instance, cleans the pid +function stop { + id=$1 + shift + if [ $id = "all" ]; then + procs=`cat $dir/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'` + # echo "stopping processes $procs" + for p in $procs; do + shutdown $p + done + rm -rf $dir/pids/* + else + pid=$dir/pids/$id.pid + if [ -f $pid ]; then + echo "stopping instance $id, pid="`cat $pid` + shutdown `cat $pid` + rm $pid + fi + fi +} + +# shutdown kills the node with interrupt 2 - if it resits falls back to -9 after 10s +function shutdown { + echo -n "stopping $1..." + kill -2 $1 + while true; do + ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break + if ((i++>5)); then + echo "not stopping. killing it" + kill -QUIT $1 + break + fi + echo '.' + sleep 1 + done + echo "stopped" +} + +# swarm restart 00 calls stop and start +function restart { + id=$1 + shift + if [ $id = "all" ]; then + stop all + N=`ls -d1 $dir/data/*|wc -l` + cluster $N $* + else + stop $id + start $id $* + fi +} + +# swarm init X sets up and starts a new client instance +########################################################## +# +# IT WIPES THE DATABASE +# +########################################################## +function init { + killall geth + reset all + cluster $* + enode all + connect all +} + +# reset wipes the datadirs of the instance +function reset { + id=$1 + shift + if [ $id = "all" ]; then + rm -rf $dir + else + rm -rf$dir/*/$id* + fi +} + +# enode displays the instance's enode address +# swarm enode all writes all instances' enodes in a file +function enode { + id=$1 + shift + + if [ $id = "all" ]; then + json=$dir/static-nodes.json + enodes=$dir/enodes.lst + cmd=$dir/connect.js + rm -f $enodes $json $cmd + # build a static nodes(-like) list of all enodes of the local cluster + echo "[" >> $json + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i $f + echo -n "admin.addPeer(" >> $cmd + cat "$f" | perl -pe 's/\s*$//' >> $cmd + echo ");" >> $cmd + cat $f |perl -pe 's/"//g'>> $enodes + cat $f >> $json + echo "," >> $json + done + echo "\"\"]" >> $json + cat $enodes + else + # echo "local IP: $ip_addr " + execute $id 'admin.nodeInfo.enode' |perl -pe "s/\[\:\:\]/$IP_ADDR/ " + fi +} + +# connect sources the local or remote set of peers and connects the node to the peers +function connect { + id=$1 + shift + if [ $id = "all" ]; then + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i $key" + download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL" +} + +# swarm up 00 file uploads file via instances CLI +function up { #port, file + echo "Upload file '$2' to node $1... " 1>&2 + file=`basename $2` + /usr/bin/time -f "latency: %e" swarm execute $1 "bzz.upload(\"$2\", \"$file\")"|tail -n1> /tmp/key + cat /tmp/key +} + +# swarm download 00 file download file via instances CLI +function download { + echo "download '$2' from node $1 to '$3'" + execute $1 "bzz.download(\"$2\", \"$3\")" >/dev/null +} + +# swarm down issues bzz.get to download the content 10 attempts +function down { + echo -n "Download hash '$2' from node $1... " + while true; do + execute $1 "bzz.get(\"$2\")" 2> /dev/null |grep -qil "status" && break + sleep 1 + echo -n "." + if ((i++>10)); then + echo "not found" + return + fi + done + echo "found OK" +} + +# static info about an instance (available even if node is off) +function info { + echo "swarm node information" + echo "ROOTDIR: $root" + echo "DATADIR: $dir/data/$1" + echo "LOGFILE: $dir/log/$1.log" + echo "HTTPAPI: http://localhost:322$1" + echo "ETHPORT: 303$1" + echo "RPCPORT: 302$1" + echo "ACCOUNT:" 0x`ls -1 $dir/data/$1/bzz` + echo "CHEQUEB:" `cat $dir/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'` + echo "ROOTDIR: $root" + echo "DATADIR: $dir/data/$1" + echo "LOGFILE: $dir/log/$1.log" +} + +# live into about an instance +function status { + echo -n "account balance: " + execute $1 'eth.getBalance(eth.accounts[0])' + echo -n "swap contract balance: " + execute $1 "eth.getBalance(bzz.info.Swap.Contract)" + echo -n "chequebook balance: " + execute $1 "chequebook.balance" + echo -n "peer count: " + execute $1 'net.peerCount' + echo -n "latest block number: " + execute $1 "eth.blockNumber" +} + +# display peers for an instance +function peers { + execute $1 'admin.peers' +} + +# add peers into an instance (connection not guaranteed) +function addpeers { + id=$1 + peers=$2 + if [ $id = "all" ]; then + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i/dev/null;echo ` + ws_server="ws://146.185.130.117:3000" + ws_secret=BZZ322 + conf="$dir/$group-$ip.netstat.json" + + echo "writing netstat conf for cluster $group-$ip ($N instances) -> $conf" + + echo -e "[" > $conf + + for ((i=0;i<$N;++i)); do + id=`printf "%02d" $i` + single_template=" {\n \"name\" : \"$group-$ip-$i\",\n \"cwd\" : \".\",\n \"script\" : \"app.js\",\n \"log_date_format\" : \"YYYY-MM-DD HH:mm Z\",\n \"merge_logs\" : false,\n \"watch\" : false,\n \"exec_interpreter\" : \"node\",\n \"exec_mode\" : \"fork_mode\",\n \"env\":\n {\n \"NODE_ENV\" : \"production\",\n \"RPC_HOST\" : \"localhost\",\n \"RPC_PORT\" : \"302$id\",\n \"INSTANCE_NAME\" : \"$group-$ip-$i\",\n \"WS_SERVER\" : \"$ws_server\",\n \"WS_SECRET\" : \"$ws_secret\",\n }\n }" + + endline="" + if ((i<$N-1)); then + # if [ "$i" -ne "$N" ]; then + endline="," + fi + echo -e "$single_template$endline" >> $conf + done + echo "]" >> $conf +} + +# (re)starts the eth-net-intelligence-api network monitor +function netstatrun { + cd $GETH_DIR/../eth-net-intelligence-api + pm2 kill + pm2 start $dir/*.netstat.json +} + +# kills the eth-net-intelligence-api network monitor +function netstatkill { + cd $GETH_DIR/../eth-net-intelligence-api + pm2 kill +} + +# copies the swarm control script to the remote node(s) +function remote-update-scripts { + scriptdir=$GETH_DIR/swarm/cmd/swarm/ + remotes=$1 + for remote in `cat $remotes|grep -v '^#'`; do echo "updating scripts on $remote..."; ssh $remote mkdir -p bin && scp -r $scriptdir/* $remote:bin/; done +} + +# copies the geth executable to the remote nodes +function remote-update-bin { + remotes=$1 + # remote-update-scripts $remotes + for remote in `cat $remotes|grep -v '^#'`; do echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done +} + +# runs a command on remote node or nodes from a file +function remote-run { + remotes=$1 + shift + if `echo "$remotes" | grep -qil @`; then + ip=`echo "$remotes"|cut -d@ -f2` + ssh $remotes "export IP_ADDR=$ip;" '. $HOME/bin/env.sh;' "$*" + else + for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; remote-run $remote "$*"; done + fi +} + +# updates the code from the given branch +function update { + branch=$1 + echo "cd $GETH_DIR && git remote update && git reset --hard $branch" + (cd $GETH_DIR && git remote update && git reset --hard $branch) +} + +function checksum { + tar -cf - $1 | md5sum|awk '{print $1}' +} + +function checkaccess { + nodes=$1 + target=`basename $2` + chsum=`md5sum $2|cut -f1 -d' '` + master=`head -1 $nodes` + echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $2|cut -f1`)" + scp $2 $master:$target + hash=`swarm remote-run $master "swarm up 00 $target $file"|tr -d '"'` + remote-run $nodes swarm checkdownload all $hash $chsum +} + +function checkdownload { + id=$1 + if [ "$id" = "all" ]; then + shift + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i /dev/null + echo + if [ -f $target ]; then + cmp --silent $target $tmpdir/$file/* && echo PASS || echo FAIL + elif [ -r $target ]; then + diff -r $target $tmpdir/$file/ >/dev/null && echo PASS || echo FAIL + else + exp=`md5sum $tmpdir/$file/*|cut -f1 -d' '` + if [ "$exp" = "$target" ]; then + echo -n PASS + else + echo FAIL "$exp = $target" + fi + fi + echo " latency: " `cat $tmpdir/$file.log` + fi +} + +function meminfo { + pid="$dir/pids/$1.pid" + if [ -f "$pid" ]; then + # cd /proc/`cat "$pid"` && cat status + ps aux|awk -v PID=`cat $pid` '$2 == PID {print $5 "Kb (" $4 "%)" }' + fi +} + +function cpuinfo { + pid="$dir/pids/$1.pid" + if [ -f "$pid" ]; then + # cd /proc/`cat "$pid"` && cat status + ps aux|awk -v PID=`cat $pid` '$2 == PID {print $3 "%" }' + fi +} + +function diskusage { + if [ "$1" == "" ]; then + echo "DISK USAGE:" `df -m |grep '/$'|awk '{print $(NF-2) "Mb (" $(NF-1) ")"} '` + else + du -m -d0 $*|cut -f1 + fi +} + +function diskinfo { + echo "DISK USAGE $1:" + echo "blockchain:" `diskusage $dir/data/$1/chaindata` + echo "chunkstore:" `diskusage $dir/data/$1/bzz/*/chunks` + echo "overall: /" `diskusage` +} + +case $cmd in + "info" ) + info $*;; + "enode" ) + enode $*;; + "status" ) + status $*;; + "peers" ) + peers $*;; + "addpeers" ) + addpeers $*;; + "clean" ) + clean $*;; + "needs" ) + needs $*;; + "up" ) + up $*;; + "key" ) + key $*;; + "down" ) + down $*;; + "download" ) + download $*;; + "init" ) + init $*;; + "exec" ) + execute $*;; + "hive" ) + hive $*;; + "start" ) + start $*;; + "stop" ) + stop $* ;; + "restart" ) + restart $*;; + "reset" ) + reset $*;; + "cluster" ) + cluster $*;; + "attach" ) + attach $*;; + "execute" ) + execute $*;; + "exec" ) + execute $*;; + "cleanbzz" ) + cleanbzz $*;; + "cleanlog" ) + cleanlog $*;; + "log" ) + log $*;; + "viewlog" ) + viewlog $*;; + "less" ) + viewlog $*;; + "connect" ) + connect $*;; + "monitor" ) + monitor $*;; + "remote-update-scripts" ) + remote-update-scripts $*;; + "remote-update-bin" ) + remote-update-bin $*;; + "update-src" ) + update-src $*;; + "remote-run" ) + remote-run $*;; + "netstatconf" ) + netstatconf $*;; + "netstatkill" ) + netstatkill $*;; + "netstatrun" ) + netstatrun $*;; + "options" ) + options $*;; + "rawoptions" ) + rawoptions $*;; + "randomfile" ) + randomfile $*;; + "diskinfo" ) + diskinfo $*;; + "meminfo" ) + meminfo $*;; + "cpuinfo" ) + cpuinfo $*;; + "setup" ) + setup $* ;; + "create-account" ) + create-account $*;; + "checkaccess" ) + checkaccess $* ;; + "checkdownload" ) + checkdownload $* ;; +esac diff --git a/swarm/cmd/swarm/swarm.sh b/swarm/cmd/swarm/swarm.sh deleted file mode 100644 index d150e25597..0000000000 --- a/swarm/cmd/swarm/swarm.sh +++ /dev/null @@ -1,316 +0,0 @@ -# !/bin/bash -# bash cluster [[params]...] -# https://github.com/ethereum/go-ethereum/wiki/Setting-up-monitoring-on-local-cluster - -# sets up a local ethereum network cluster of nodes -# - is the number of nodes in cluster -# - is the root directory for the cluster, the nodes are set up -# with datadir `//00`, `/ /01`, ... -# - new accounts are created for each node -# - they launch on port 30300, 30301, ... -# - they star rpc on port 8100, 8101, ... -# - by collecting the nodes nodeUrl, they get connected to each other -# - if enode has no IP, `` is substituted -# - if `` is not 0, they will not connect to a default client, -# resulting in a private isolated network -# - the nodes log into `//00..log`, `//01..log`, ... -# - The nodes launch in mining mode -# - the cluster can be killed with `killall geth` (FIXME: should record PIDs) -# and restarted from the same state -# - if you want to interact with the nodes, use rpc -# - you can supply additional params on the command line which will be passed -# to each node, for instance `-mine` - -if [ "$GETH" = "" ]; then - echo "env var GETH not set " - exit 1 -fi - -srcdir=`dirname $0` - -root=$1 -shift -network_id=$1 -shift -cmd=$1 -shift -# ip_addr=`curl ipecho.net/plain 2>/dev/null;echo ` - -# echo "external IP: $ip_addr" -swarmoptions='--dev --maxpeers=20 --shh=false --nodiscover' -tmpdir=/tmp - -function attach { - id=$1 - shift - echo "attaching console to instance $id" - cmd="$GETH $* attach ipc:$root/$network_id/data/$id/geth.ipc" - # echo $cmd - eval $cmd -} - -function log { - id=$1 - shift - echo "streaming logs for instance $id" - cmd="tail -f $root/$network_id/log/$id.log" - echo $cmd - eval $cmd -} - -function less { - id=$1 - shift - echo "viewing logs for instance $id" - cmd="/usr/bin/less $root/$network_id/log/$id.log" - echo $cmd - eval $cmd -} - -function start { - id=$1 - shift - # echo -n "starting instance $id - " - cmd="bash $srcdir/gethup.sh $root/$network_id/ $id '$ip_addr' --networkid=$network_id $swarmoptions $*" - # echo "pid="`cat $root/$network_id/pids/$id.pid` - # echo $cmd - eval $cmd -} - -function stop { - id=$1 - shift - if [ $id = "all" ]; then - procs=`cat $root/$network_id/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'` - # echo "stopping processes $procs" - for p in $procs; do - shutdown $p - done - rm -rf $root/$network_id/pids/* - else - pid=$root/$network_id/pids/$id.pid - if [ -f $pid ]; then - echo "stopping instance $id, pid="`cat $pid` - shutdown `cat $pid` - rm $pid - fi - fi - # ps auxwww|grep geth|grep bzz|grep -v grep -} - -function shutdown { - echo -n "stopping $1..." - kill -2 $1 - while true ;do - ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break - sleep 1 - # ps auxwww|grep geth|grep -v grep |grep $1 #|awk '{print $2}' - done - echo "stopped" -} - -function restart { - id=$1 - shift - stop $id - start $id $* -} - -function init { - stop all - killall geth - reset all - cluster $* - stop all - cluster $* -} - -function reset { - id=$1 - shift - if [ $id = "all" ]; then - rm -rf $root/$network_id - else - rm -rf$root/$network_id/*/$id* - fi - -} - -function cluster { - N=$1 - shift - echo "launching cluster of $N instances" - # cmd="bash $srcdir/gethcluster.sh $root $network_id $N '' $swarmoptions $*" - # echo $cmd - # eval $cmd - dir=$root/$network_id - mkdir -p $dir/data - mkdir -p $dir/enodes - mkdir -p $dir/pids - mkdir -p $dir/log - - enodes=$dir/enodes.all - rm -f $enodes - # build a static nodes(-like) list of all enodes of the local cluster - echo "[" >> $enodes - for ((i=0;i> $enodes - echo "," >> $enodes - fi - done - echo "\"\"]" >> $enodes - - for ((i=0;i tail -f $dir/log/$id.log" - start $id $vmodule $* - done -} - - -function needs { - id=$1 - keyfile=$2 - target=$3 - dir=`dirname $3` - dest=$tmpdir/down - mkdir -p $dest - file=$dest/`basename $target` - rm -f $file - echo -n "waiting for root hash in '$keyfile'..." - while true; do - if [ -f $keyfile ] && [ ! -z $keyfile ]; then - break - fi - sleep 1 - echo -n "." - done - key=`cat $keyfile|tr -d \"` - echo " => $key" - download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL" - # && ls -l $keyfile $file $target -} - - -function up { #port, file - echo "Upload file '$2' to node $1... " 1>&2 - file=`basename $2` - attach $1 "--exec 'bzz.upload(\"$2\", \"$file\")'"|tail -n1> /tmp/key - # key=`bash swarm/cmd/bzzup.sh $2 86$1` - cat /tmp/key -} - -function download { - echo "download '$2' from node $1 to '$3'" - # echo attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" - attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" > /dev/null -} - - -function down { - echo -n "Download hash '$2' from node $1... " - # echo "wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo 'got it' || echo 'not found'" - # wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo "got it" || echo "not found" - while true; do - attach $1 "--exec 'bzz.get(\"$2\")'" 2> /dev/null |grep -qil "status" && break - sleep 1 - echo -n "." - if ((i++>10)); then - echo "not found" - return - fi - done - echo "found OK" -} - -function clean { #index - echo "Clean up for $1" - rm -rf $root/$network_id/data/$1/{bzz/*/chunks,bzz/*/requests/,bzz/*/bzz-peers.json,chaindata,nodes} -} - -function info { - echo "swarm node information" - echo "ROOTDIR: $root" - echo "DATADIR: $root/$network_id/data/$1" - echo "LOGFILE: $root/$network_id/log/$1.log" - echo "HTTPAPI: http://localhost:322$1" - echo "ETHPORT: 303$1" - echo "RPCPORT: 302$1" - echo "ACCOUNT:" 0x`ls -1 $root/$network_id/data/$1/bzz` - echo "CHEQUEB:" `cat $root/$network_id/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'` - echo "ROOTDIR: $root" - echo "DATADIR: $root/$network_id/data/$1" - echo "LOGFILE: $root/$network_id/log/$1.log" -} - - -function status { - attach 00 -exec "'console.log(eth.getBalance(eth.accounts[0])); console.log(eth.getBalance(bzz.info().Swap.Contract)); console.log(chequebook.balance)'" -} - -function netstatconf { - begin=$1 - N=$2 - name_prefix=$3 - ws_server=$4 - ws_secret=$5 - conf="$root/$network_id/$name_prefix.netstat.json" - - echo "writing netstat conf for cluster $name_prefix to $conf" - - echo -e "[" > $conf - - for ((i=$begin;i<$start+$N;++i)); do - id=`printf "%02d" $i` - single_template=" {\n \"name\" : \"$name_prefix-$i\",\n \"cwd\" : \".\",\n \"script\" : \"app.js\",\n \"log_date_format\" : \"YYYY-MM-DD HH:mm Z\",\n \"merge_logs\" : false,\n \"watch\" : false,\n \"exec_interpreter\" : \"node\",\n \"exec_mode\" : \"fork_mode\",\n \"env\":\n {\n \"NODE_ENV\" : \"production\",\n \"RPC_HOST\" : \"localhost\",\n \"RPC_PORT\" : \"302$id\",\n \"INSTANCE_NAME\" : \"$name_prefix-$i\",\n \"WS_SERVER\" : \"$ws_server\",\n \"WS_SECRET\" : \"$ws_secret\",\n }\n }" - - endline="" - if (($i<$N-1)); then - # if [ "$i" -ne "$N" ]; then - endline="," - fi - echo -e "$single_template$endline" >> $conf - done - - echo "]" >> $conf -} - -case $cmd in - "info" ) - info $*;; - "status" ) - status $*;; - "clean" ) - clean $*;; - "needs" ) - needs $*;; - "up" ) - up $*;; - "down" ) - down $*;; - "init" ) - init $*;; - "start" ) - start $*;; - "stop" ) - stop $* ;; - "restart" ) - restart $*;; - "reset" ) - reset $*;; - "cluster" ) - cluster $*;; - "attach" ) - attach $*;; - "log" ) - log $*;; - "less" ) - less $*;; - "netstatconf" ) - netstatconf $*;; - -esac diff --git a/swarm/cmd/swarm/test.sh b/swarm/cmd/swarm/test.sh deleted file mode 100644 index 5ded17493a..0000000000 --- a/swarm/cmd/swarm/test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -TEST_DIR=`dirname $0` -TEST_NAME=`basename $0 .sh` -TEST_TYPE=`basename $TEST_DIR` - - -export SWARM_BIN=$TEST_DIR/../../cmd/swarm -export GETH=$SWARM_BIN/../../../geth -export NETWORKID=322$TEST_NAME -export TMPDIR=~/BZZ/test/$TEST_TYPE -export DATA_ROOT=$TMPDIR/$NETWORKID -# alias swarm='bash $SWARM_BIN/swarm.sh $DATA_ROOT $NETWORKID' -EXTRA_ARGS=$* - -rm -rf $DATA_ROOT - -wait=1 - -function swarm { - # echo bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS - bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS -} - - -function randomfile { - dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null -} \ No newline at end of file diff --git a/swarm/examples/album/file-manager.js b/swarm/examples/album/file-manager.js index 3e3f60deb6..d2a024721e 100644 --- a/swarm/examples/album/file-manager.js +++ b/swarm/examples/album/file-manager.js @@ -103,13 +103,14 @@ function onUploadingComplete(uri) { xhr.onreadystatechange = function () { if (xhr.readyState === 4) { var i = xhr.responseText; - window.location.replace("/bzz:/" + i + "/"); + window.location = "/bzz:/" + i + "/" + window.location.hash; } }; sendImages(xhr, uri); } function handleFiles(files) { + showModal('Uploading photos..'); uploadFile(files, 0, ""); } diff --git a/swarm/examples/album/index.html b/swarm/examples/album/index.html index 0f73c72798..64fcdc67a0 100644 --- a/swarm/examples/album/index.html +++ b/swarm/examples/album/index.html @@ -5,6 +5,7 @@ + @@ -16,8 +17,15 @@ + + + + diff --git a/swarm/examples/album/index.js b/swarm/examples/album/index.js index c1dd51771b..9fbbcc311c 100644 --- a/swarm/examples/album/index.js +++ b/swarm/examples/album/index.js @@ -2,8 +2,7 @@ // Copyright(c) 2003-2014 by wave++ "Yuri D'Elia" // Distributed under GPL2 (see COPYING) WITHOUT ANY WARRANTY. var datafile = 'data.json'; -var padding = 100; -var marginTop = 50; +var padding = 20; var duration = 500; var thrdelay = 1500; var hidedelay = 3000; @@ -42,7 +41,6 @@ var eback; // background var enoise; // additive noise var eflash; // flashing object var ehdr; // header -var progress; // progress var elist; // thumbnail list var fscr; // thumbnail list scroll fx var econt; // picture container @@ -61,6 +59,18 @@ var idle; // idle timer var clayout; // current layout var csr; // current scaling ratio +function showModal(text, previewImageUrl) { + if (!previewImageUrl) { + previewImageUrl = 'throbber.gif'; + } + + jQuery('#currentPreview').attr('src', previewImageUrl); + jQuery('#action-text').text(text); + jQuery('#preview-modal').modal({ + fadeDuration: 250 + }); +} + function resize() { // best layout @@ -233,7 +243,7 @@ function resizeMainImg(img) img.setStyles( { 'position': 'absolute', - 'top': (contSize.y / 2 - img.height / 2) + marginTop, + 'top': contSize.y / 2 - img.height / 2, 'left': contSize.x / 2 - img.width / 2 }); } @@ -275,41 +285,53 @@ function imageToUrl(img, w, h) { return can.toDataURL(); } -function deleteImg() -{ - if(imgs.data.length < 2) return; // empty albums not allowed - var fname = imgs.data[eidx].img[0]; - imgs.data.splice(eidx,1); +function deleteImg() { + if (imgs.data.length < 2) return; // empty albums not allowed + var fname = imgs.data[eidx].img[0]; + imgs.data.splice(eidx, 1); - // construct an HTTP request - var xhr = new XMLHttpRequest(); + showModal('Deleting photo..', fname); + // construct an HTTP request + var xhr = new XMLHttpRequest(); - // set response handler - xhr.onreadystatechange = function () { if (xhr.readyState === 4) { - var i = xhr.responseText; - var xhrd = new XMLHttpRequest(); - xhrd.onreadystatechange = function () { if (xhrd.readyState === 4) { - var j = xhrd.responseText; - window.location.replace("/bzz:/" + j + "/"); - }}; - xhrd.open("DELETE", "/bzz%3A/" + i + "/" + fname, true); - xhrd.send(); - }}; + // set response handler + xhr.onreadystatechange = function () { + if (xhr.readyState === 4) { + var i = xhr.responseText; + var xhrd = new XMLHttpRequest(); + xhrd.onreadystatechange = function () { + if (xhrd.readyState === 4) { + var j = xhrd.responseText; + window.location = "/bzz:/" + j + "/" + window.location.hash; + } + }; + xhrd.open("DELETE", "/bzz%3A/" + i + "/" + fname, true); + xhrd.send(); + } + }; - sendImages(xhr, ""); + sendImages(xhr, ""); } -function moveUpDown(off) -{ - var me = imgs.data[eidx]; - imgs.data[eidx] = imgs.data[eidx + off]; - imgs.data[eidx + off] = me; - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function () { if (xhr.readyState === 4) { - var i = xhr.responseText; - window.location.replace("/bzz:/" + i + "/#" + (eidx + off)); - }}; - sendImages(xhr, ""); +function moveUpDown(off) { + var me = imgs.data[eidx]; + console.log(me.thumb[0]); + var moveText = 'Moving up..'; + if (off > 0) { + moveText = 'Moving down..'; + } + + showModal(moveText, me.thumb[0]); + imgs.data[eidx] = imgs.data[eidx + off]; + imgs.data[eidx + off] = me; + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4) { + var i = xhr.responseText; + window.location.replace("/bzz:/" + i + "/#" + (eidx + off)); + } + }; + sendImages(xhr, ""); } function moveUp() @@ -364,9 +386,6 @@ function onMainReady() ehdr.set('html', dsc.join(' ')); ehdr.setStyle('display', (dsc.length? 'block': 'none')); - progress.set('html', ''); - progress.set('style', 'text-align: center; padding-top: 20px'); - // complete thumbnails var d = duration; if(first !== false) @@ -601,9 +620,6 @@ function initGallery(data) ehdr = new Element('div', { id: 'header' }); ehdr.inject(econt); - progress = new Element('div', { id: 'progress' }); - progress.inject(econt); - elist = new Element('div', { id: 'list' }); elist.inject(emain); diff --git a/swarm/examples/album/jquery.modal.css b/swarm/examples/album/jquery.modal.css new file mode 100644 index 0000000000..0da4866070 --- /dev/null +++ b/swarm/examples/album/jquery.modal.css @@ -0,0 +1,70 @@ +.blocker { + position: fixed; + top: 0; right: 0; bottom: 0; left: 0; + width: 100%; height: 100%; + overflow: auto; + z-index: 1; + padding: 20px; + box-sizing: border-box; + background-color: rgb(0,0,0); + background-color: rgba(0,0,0,0.75); + text-align: center; +} +.blocker:before{ + content: ""; + display: inline-block; + height: 100%; + vertical-align: middle; + margin-right: -0.05em; +} +.blocker.behind { + background-color: transparent; +} +.modal { + display: inline-block; + vertical-align: middle; + position: relative; + z-index: 2; + width: 400px; + background: #fff; + padding: 15px 30px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; + -o-border-radius: 8px; + -ms-border-radius: 8px; + border-radius: 8px; + -webkit-box-shadow: 0 0 10px #000; + -moz-box-shadow: 0 0 10px #000; + -o-box-shadow: 0 0 10px #000; + -ms-box-shadow: 0 0 10px #000; + box-shadow: 0 0 10px #000; + text-align: left; +} + +.modal a.close-modal { + position: absolute; + top: -12.5px; + right: -12.5px; + display: block; + width: 30px; + height: 30px; + text-indent: -9999px; + background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAAeCAYAAAA7MK6iAAAACXBIWXMAAAsTAAALEwEAmpwYAAAABGdBTUEAANjr9RwUqgAAACBjSFJNAABtmAAAc44AAPJxAACDbAAAg7sAANTIAAAx7AAAGbyeiMU/AAAG7ElEQVR42mJkwA8YoZjBwcGB6fPnz4w/fvxg/PnzJ2N6ejoLFxcX47Rp036B5Dk4OP7z8vL+P3DgwD+o3v9QjBUABBALHguZoJhZXV2dVUNDgxNIcwEtZnn27Nl/ZmZmQRYWFmag5c90dHQY5OXl/z98+PDn1atXv79+/foPUN9fIP4HxRgOAAggRhyWMoOwqKgoq6GhIZe3t7eYrq6uHBDb8/Pz27Gysloga/jz588FYGicPn/+/OapU6deOnXq1GdgqPwCOuA31AF/0S0HCCB0xAQNBU4FBQWB0NBQublz59oADV37Hw28ePHi74MHD/6ii3/8+HEFMGQUgQ6WEhQU5AeZBTWTCdkigABC9ylIAZeMjIxQTEyMysaNG/3+/v37AGTgr1+//s2cOfOXm5vbN6Caz8jY1NT0a29v76/v37//g6q9sHfv3khjY2M5YAgJgsyEmg0PYYAAQreUk4+PT8jd3V1l1apVgUAzfoIM2rlz5x9gHH5BtxAdA9PB1zNnzvyB+R6oLxoopgC1nBPZcoAAgiFQnLIDMb+enp5iV1eXBzDeHoI0z58//xcwIX0mZCkMg9S2trb+hFk+ffr0QCkpKVmQ2VA7QHYxAgQQzLesQMwjIiIilZWVZfPu3bstMJ+SYikyBmUzkBnA9HEMyNcCYgmQHVC7mAACCJagOEBBbGdnp7lgwYJEkIavX7/+BcY1SvAaGRl9tba2xohjMTGxL8nJyT+AWQsuxsbG9vnp06e/QWYdPHiwHmiWKlBcCGQXyNcAAQSzmBuoSQqYim3u37+/EKR48uTJv5ANB+bVr7Dga2xs/AkTV1JS+gq0AJyoQIkPWU9aWtoPkPibN2/2A/l6QCwJ9TULQADB4hcY//xKXl5eHt++fbsAUmxhYYHiM1DiAsr9R7ZcVVUVbikIdHd3/0TWIyws/AWYVsByAgICdkAxRSAWAGI2gACClV7C4uLiOv7+/lEgRZ8+ffqLLd6ABck3ZMuB6uCWrlu37je29HDx4kVwQisvL88FFqkaQDERUHADBBAomBl5eHiYgQmLE1hSgQQZgIUD1lJm69atf4HR8R1YKoH5QIPAWWP9+vV/gOI/gHkeQw+wGAXTwAJJ5t+/f/BUDRBA4NIEKMDMyMjICtQIiniG379/4yza7t69+//Lly8oDrty5co/bJaCAEwcZCkwwTJDLWYCCCCwxcDgY3z16hXDnTt3voP4EhISWA0BFgZMwNqHExh3jMiG1tbWsgHjnA2bHmAeBtdWwOL1MycnJ7wAAQggBmi+kgIW/OaKiorJwOLuFShO0LMSMPF9AUYBSpz6+vqixHlOTs4P9MIEWHaDsxSwYMoE2mEGFJcG5SKAAGJCqjv/AbPUn8ePH98ACQQHB6NUmZqamkzABIgSp5s3bwbHORCA1QDLAWZkPc7OzszA8oHl5cuXVy5duvQBGIXwWgoggGA+FgO6xkBNTS28r69vDrT2+Y1cIMDyJchX6KkXVEmAshd6KB06dAic94EO3AzkBwGxPhCLg8ptgACCZyeQp9jZ2b2AmsuAefM8tnxJCk5ISPgOLTKfAdNEOVDMA2QHLDsBBBC8AAFlbmCLwlZISCg5JSVlJizeQAaQaimoWAUFK0g/sGGwHiiWCMS2yAUIQAAxI7c4gEmeFZi4OJ48ecLMzc39CRiEmgEBASxA/QzA8vYvAxEgNjaWZc2aNezAsprp2LFjp4FpZRdQ+AkQvwLij0AMSoC/AQIIXklAC3AVUBoBxmE8sPXQAiyvN8J8fuPGjR/h4eHf0eMdhkENhOPHj8OT+NGjR88BxZuBOA5kJtRseCUBEECMSI0AdmgBDooDaaDl8sASTSkyMlKzpqZGU1paGlS7MABLrX83b978A6zwwakTmE0YgIkSnHpBfGCV+gxYh98qKSk5CeTeAxVeQPwUiN8AMSjxgdLNX4AAYkRqCLBAXcMHtVwSaLkMMMHJAvOq9IQJE9R8fHxElJWV1bEF8aNHj+7t27fvLTDlXwXGLyhoH0OD+DnU0k/QYAa1QP8BBBAjWsuSFWo5LzRYxKFYAljqiAHzqxCwIBEwMTERBdZeoOYMA7Bl+RFYEbwB5oS3IA9D4/IFEL+E4nfQ6IDFLTgvAwQQI5ZmLRtSsINSuyA0uwlBUyQPMPWD20/AKo8ByP4DTJTfgRgUjB+gFoEc8R6amGDB+wu5mQsQQIxYmrdMUJ+zQTM6NzQEeKGO4UJqOzFADQMZ/A1qCSzBfQXi71ALfyM17sEAIIAY8fQiWKAYFgIwzIbWTv4HjbdfUAf8RPLhH1icojfoAQKIEU8bG9kRyF0aRiz6YP0k5C4LsmUY9TtAADEyEA+IVfufGEUAAQYABejinPr4dLEAAAAASUVORK5CYII=") no-repeat 0 0; +} + +.modal-spinner { + display: none; + width: 64px; + height: 64px; + position: fixed; + top: 50%; + left: 50%; + margin-right: -32px; + margin-top: -32px; + background: url("data:image/gif;base64,R0lGODlhIAAgAPMAABEREf///0VFRYKCglRUVG5ubsvLy62trTQ0NCkpKU5OTuLi4vr6+gAAAAAAAAAAACH+GkNyZWF0ZWQgd2l0aCBhamF4bG9hZC5pbmZvACH5BAAKAAAAIf8LTkVUU0NBUEUyLjADAQAAACwAAAAAIAAgAAAE5xDISWlhperN52JLhSSdRgwVo1ICQZRUsiwHpTJT4iowNS8vyW2icCF6k8HMMBkCEDskxTBDAZwuAkkqIfxIQyhBQBFvAQSDITM5VDW6XNE4KagNh6Bgwe60smQUB3d4Rz1ZBApnFASDd0hihh12BkE9kjAJVlycXIg7CQIFA6SlnJ87paqbSKiKoqusnbMdmDC2tXQlkUhziYtyWTxIfy6BE8WJt5YJvpJivxNaGmLHT0VnOgSYf0dZXS7APdpB309RnHOG5gDqXGLDaC457D1zZ/V/nmOM82XiHRLYKhKP1oZmADdEAAAh+QQACgABACwAAAAAIAAgAAAE6hDISWlZpOrNp1lGNRSdRpDUolIGw5RUYhhHukqFu8DsrEyqnWThGvAmhVlteBvojpTDDBUEIFwMFBRAmBkSgOrBFZogCASwBDEY/CZSg7GSE0gSCjQBMVG023xWBhklAnoEdhQEfyNqMIcKjhRsjEdnezB+A4k8gTwJhFuiW4dokXiloUepBAp5qaKpp6+Ho7aWW54wl7obvEe0kRuoplCGepwSx2jJvqHEmGt6whJpGpfJCHmOoNHKaHx61WiSR92E4lbFoq+B6QDtuetcaBPnW6+O7wDHpIiK9SaVK5GgV543tzjgGcghAgAh+QQACgACACwAAAAAIAAgAAAE7hDISSkxpOrN5zFHNWRdhSiVoVLHspRUMoyUakyEe8PTPCATW9A14E0UvuAKMNAZKYUZCiBMuBakSQKG8G2FzUWox2AUtAQFcBKlVQoLgQReZhQlCIJesQXI5B0CBnUMOxMCenoCfTCEWBsJColTMANldx15BGs8B5wlCZ9Po6OJkwmRpnqkqnuSrayqfKmqpLajoiW5HJq7FL1Gr2mMMcKUMIiJgIemy7xZtJsTmsM4xHiKv5KMCXqfyUCJEonXPN2rAOIAmsfB3uPoAK++G+w48edZPK+M6hLJpQg484enXIdQFSS1u6UhksENEQAAIfkEAAoAAwAsAAAAACAAIAAABOcQyEmpGKLqzWcZRVUQnZYg1aBSh2GUVEIQ2aQOE+G+cD4ntpWkZQj1JIiZIogDFFyHI0UxQwFugMSOFIPJftfVAEoZLBbcLEFhlQiqGp1Vd140AUklUN3eCA51C1EWMzMCezCBBmkxVIVHBWd3HHl9JQOIJSdSnJ0TDKChCwUJjoWMPaGqDKannasMo6WnM562R5YluZRwur0wpgqZE7NKUm+FNRPIhjBJxKZteWuIBMN4zRMIVIhffcgojwCF117i4nlLnY5ztRLsnOk+aV+oJY7V7m76PdkS4trKcdg0Zc0tTcKkRAAAIfkEAAoABAAsAAAAACAAIAAABO4QyEkpKqjqzScpRaVkXZWQEximw1BSCUEIlDohrft6cpKCk5xid5MNJTaAIkekKGQkWyKHkvhKsR7ARmitkAYDYRIbUQRQjWBwJRzChi9CRlBcY1UN4g0/VNB0AlcvcAYHRyZPdEQFYV8ccwR5HWxEJ02YmRMLnJ1xCYp0Y5idpQuhopmmC2KgojKasUQDk5BNAwwMOh2RtRq5uQuPZKGIJQIGwAwGf6I0JXMpC8C7kXWDBINFMxS4DKMAWVWAGYsAdNqW5uaRxkSKJOZKaU3tPOBZ4DuK2LATgJhkPJMgTwKCdFjyPHEnKxFCDhEAACH5BAAKAAUALAAAAAAgACAAAATzEMhJaVKp6s2nIkolIJ2WkBShpkVRWqqQrhLSEu9MZJKK9y1ZrqYK9WiClmvoUaF8gIQSNeF1Er4MNFn4SRSDARWroAIETg1iVwuHjYB1kYc1mwruwXKC9gmsJXliGxc+XiUCby9ydh1sOSdMkpMTBpaXBzsfhoc5l58Gm5yToAaZhaOUqjkDgCWNHAULCwOLaTmzswadEqggQwgHuQsHIoZCHQMMQgQGubVEcxOPFAcMDAYUA85eWARmfSRQCdcMe0zeP1AAygwLlJtPNAAL19DARdPzBOWSm1brJBi45soRAWQAAkrQIykShQ9wVhHCwCQCACH5BAAKAAYALAAAAAAgACAAAATrEMhJaVKp6s2nIkqFZF2VIBWhUsJaTokqUCoBq+E71SRQeyqUToLA7VxF0JDyIQh/MVVPMt1ECZlfcjZJ9mIKoaTl1MRIl5o4CUKXOwmyrCInCKqcWtvadL2SYhyASyNDJ0uIiRMDjI0Fd30/iI2UA5GSS5UDj2l6NoqgOgN4gksEBgYFf0FDqKgHnyZ9OX8HrgYHdHpcHQULXAS2qKpENRg7eAMLC7kTBaixUYFkKAzWAAnLC7FLVxLWDBLKCwaKTULgEwbLA4hJtOkSBNqITT3xEgfLpBtzE/jiuL04RGEBgwWhShRgQExHBAAh+QQACgAHACwAAAAAIAAgAAAE7xDISWlSqerNpyJKhWRdlSAVoVLCWk6JKlAqAavhO9UkUHsqlE6CwO1cRdCQ8iEIfzFVTzLdRAmZX3I2SfZiCqGk5dTESJeaOAlClzsJsqwiJwiqnFrb2nS9kmIcgEsjQydLiIlHehhpejaIjzh9eomSjZR+ipslWIRLAgMDOR2DOqKogTB9pCUJBagDBXR6XB0EBkIIsaRsGGMMAxoDBgYHTKJiUYEGDAzHC9EACcUGkIgFzgwZ0QsSBcXHiQvOwgDdEwfFs0sDzt4S6BK4xYjkDOzn0unFeBzOBijIm1Dgmg5YFQwsCMjp1oJ8LyIAACH5BAAKAAgALAAAAAAgACAAAATwEMhJaVKp6s2nIkqFZF2VIBWhUsJaTokqUCoBq+E71SRQeyqUToLA7VxF0JDyIQh/MVVPMt1ECZlfcjZJ9mIKoaTl1MRIl5o4CUKXOwmyrCInCKqcWtvadL2SYhyASyNDJ0uIiUd6GGl6NoiPOH16iZKNlH6KmyWFOggHhEEvAwwMA0N9GBsEC6amhnVcEwavDAazGwIDaH1ipaYLBUTCGgQDA8NdHz0FpqgTBwsLqAbWAAnIA4FWKdMLGdYGEgraigbT0OITBcg5QwPT4xLrROZL6AuQAPUS7bxLpoWidY0JtxLHKhwwMJBTHgPKdEQAACH5BAAKAAkALAAAAAAgACAAAATrEMhJaVKp6s2nIkqFZF2VIBWhUsJaTokqUCoBq+E71SRQeyqUToLA7VxF0JDyIQh/MVVPMt1ECZlfcjZJ9mIKoaTl1MRIl5o4CUKXOwmyrCInCKqcWtvadL2SYhyASyNDJ0uIiUd6GAULDJCRiXo1CpGXDJOUjY+Yip9DhToJA4RBLwMLCwVDfRgbBAaqqoZ1XBMHswsHtxtFaH1iqaoGNgAIxRpbFAgfPQSqpbgGBqUD1wBXeCYp1AYZ19JJOYgH1KwA4UBvQwXUBxPqVD9L3sbp2BNk2xvvFPJd+MFCN6HAAIKgNggY0KtEBAAh+QQACgAKACwAAAAAIAAgAAAE6BDISWlSqerNpyJKhWRdlSAVoVLCWk6JKlAqAavhO9UkUHsqlE6CwO1cRdCQ8iEIfzFVTzLdRAmZX3I2SfYIDMaAFdTESJeaEDAIMxYFqrOUaNW4E4ObYcCXaiBVEgULe0NJaxxtYksjh2NLkZISgDgJhHthkpU4mW6blRiYmZOlh4JWkDqILwUGBnE6TYEbCgevr0N1gH4At7gHiRpFaLNrrq8HNgAJA70AWxQIH1+vsYMDAzZQPC9VCNkDWUhGkuE5PxJNwiUK4UfLzOlD4WvzAHaoG9nxPi5d+jYUqfAhhykOFwJWiAAAIfkEAAoACwAsAAAAACAAIAAABPAQyElpUqnqzaciSoVkXVUMFaFSwlpOCcMYlErAavhOMnNLNo8KsZsMZItJEIDIFSkLGQoQTNhIsFehRww2CQLKF0tYGKYSg+ygsZIuNqJksKgbfgIGepNo2cIUB3V1B3IvNiBYNQaDSTtfhhx0CwVPI0UJe0+bm4g5VgcGoqOcnjmjqDSdnhgEoamcsZuXO1aWQy8KAwOAuTYYGwi7w5h+Kr0SJ8MFihpNbx+4Erq7BYBuzsdiH1jCAzoSfl0rVirNbRXlBBlLX+BP0XJLAPGzTkAuAOqb0WT5AH7OcdCm5B8TgRwSRKIHQtaLCwg1RAAAOwAAAAAAAAAAAA==") #111 no-repeat center center; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; + -o-border-radius: 8px; + -ms-border-radius: 8px; + border-radius: 8px; +} diff --git a/swarm/examples/album/jquery.modal.js b/swarm/examples/album/jquery.modal.js new file mode 100644 index 0000000000..40c4c7a937 --- /dev/null +++ b/swarm/examples/album/jquery.modal.js @@ -0,0 +1,228 @@ +/* + A simple jQuery modal (http://github.com/kylefox/jquery-modal) + Version 0.7.0 +*/ +(function($) { + + var modals = [], + getCurrent = function() { + return modals.length ? modals[modals.length - 1] : null; + }, + selectCurrent = function() { + var i, + selected = false; + for (i=modals.length-1; i>=0; i--) { + if (modals[i].$blocker) { + modals[i].$blocker.toggleClass('current',!selected).toggleClass('behind',selected); + selected = true; + } + } + }; + + $.modal = function(el, options) { + var remove, target; + this.$body = $('body'); + this.options = $.extend({}, $.modal.defaults, options); + this.options.doFade = !isNaN(parseInt(this.options.fadeDuration, 10)); + this.$blocker = null; + if (this.options.closeExisting) + while ($.modal.isActive()) + $.modal.close(); // Close any open modals. + modals.push(this); + if (el.is('a')) { + target = el.attr('href'); + //Select element by id from href + if (/^#/.test(target)) { + this.$elm = $(target); + if (this.$elm.length !== 1) return null; + this.$body.append(this.$elm); + this.open(); + //AJAX + } else { + this.$elm = $('
'); + this.$body.append(this.$elm); + remove = function(event, modal) { modal.elm.remove(); }; + this.showSpinner(); + el.trigger($.modal.AJAX_SEND); + $.get(target).done(function(html) { + if (!$.modal.isActive()) return; + el.trigger($.modal.AJAX_SUCCESS); + var current = getCurrent(); + current.$elm.empty().append(html).on($.modal.CLOSE, remove); + current.hideSpinner(); + current.open(); + el.trigger($.modal.AJAX_COMPLETE); + }).fail(function() { + el.trigger($.modal.AJAX_FAIL); + var current = getCurrent(); + current.hideSpinner(); + modals.pop(); // remove expected modal from the list + el.trigger($.modal.AJAX_COMPLETE); + }); + } + } else { + this.$elm = el; + this.$body.append(this.$elm); + this.open(); + } + }; + + $.modal.prototype = { + constructor: $.modal, + + open: function() { + var m = this; + this.block(); + if(this.options.doFade) { + setTimeout(function() { + m.show(); + }, this.options.fadeDuration * this.options.fadeDelay); + } else { + this.show(); + } + $(document).off('keydown.modal').on('keydown.modal', function(event) { + var current = getCurrent(); + if (event.which == 27 && current.options.escapeClose) current.close(); + }); + if (this.options.clickClose) + this.$blocker.click(function(e) { + if (e.target==this) + $.modal.close(); + }); + }, + + close: function() { + modals.pop(); + this.unblock(); + this.hide(); + if (!$.modal.isActive()) + $(document).off('keydown.modal'); + }, + + block: function() { + this.$elm.trigger($.modal.BEFORE_BLOCK, [this._ctx()]); + this.$body.css('overflow','hidden'); + this.$blocker = $('
').appendTo(this.$body); + selectCurrent(); + if(this.options.doFade) { + this.$blocker.css('opacity',0).animate({opacity: 1}, this.options.fadeDuration); + } + this.$elm.trigger($.modal.BLOCK, [this._ctx()]); + }, + + unblock: function(now) { + if (!now && this.options.doFade) + this.$blocker.fadeOut(this.options.fadeDuration, this.unblock.bind(this,true)); + else { + this.$blocker.children().appendTo(this.$body); + this.$blocker.remove(); + this.$blocker = null; + selectCurrent(); + if (!$.modal.isActive()) + this.$body.css('overflow',''); + } + }, + + show: function() { + this.$elm.trigger($.modal.BEFORE_OPEN, [this._ctx()]); + if (this.options.showClose) { + this.closeButton = $('' + this.options.closeText + ''); + this.$elm.append(this.closeButton); + } + this.$elm.addClass(this.options.modalClass).appendTo(this.$blocker); + if(this.options.doFade) { + this.$elm.css('opacity',0).show().animate({opacity: 1}, this.options.fadeDuration); + } else { + this.$elm.show(); + } + this.$elm.trigger($.modal.OPEN, [this._ctx()]); + }, + + hide: function() { + this.$elm.trigger($.modal.BEFORE_CLOSE, [this._ctx()]); + if (this.closeButton) this.closeButton.remove(); + var _this = this; + if(this.options.doFade) { + this.$elm.fadeOut(this.options.fadeDuration, function () { + _this.$elm.trigger($.modal.AFTER_CLOSE, [_this._ctx()]); + }); + } else { + this.$elm.hide(0, function () { + _this.$elm.trigger($.modal.AFTER_CLOSE, [_this._ctx()]); + }); + } + this.$elm.trigger($.modal.CLOSE, [this._ctx()]); + }, + + showSpinner: function() { + if (!this.options.showSpinner) return; + this.spinner = this.spinner || $('') + .append(this.options.spinnerHtml); + this.$body.append(this.spinner); + this.spinner.show(); + }, + + hideSpinner: function() { + if (this.spinner) this.spinner.remove(); + }, + + //Return context for custom events + _ctx: function() { + return { elm: this.$elm, $blocker: this.$blocker, options: this.options }; + } + }; + + $.modal.close = function(event) { + if (!$.modal.isActive()) return; + if (event) event.preventDefault(); + var current = getCurrent(); + current.close(); + return current.$elm; + }; + + // Returns if there currently is an active modal + $.modal.isActive = function () { + return modals.length > 0; + } + + $.modal.defaults = { + closeExisting: true, + escapeClose: true, + clickClose: true, + closeText: 'Close', + closeClass: '', + modalClass: "modal", + spinnerHtml: null, + showSpinner: true, + showClose: true, + fadeDuration: null, // Number of milliseconds the fade animation takes. + fadeDelay: 1.0 // Point during the overlay's fade-in that the modal begins to fade in (.5 = 50%, 1.5 = 150%, etc.) + }; + + // Event constants + $.modal.BEFORE_BLOCK = 'modal:before-block'; + $.modal.BLOCK = 'modal:block'; + $.modal.BEFORE_OPEN = 'modal:before-open'; + $.modal.OPEN = 'modal:open'; + $.modal.BEFORE_CLOSE = 'modal:before-close'; + $.modal.CLOSE = 'modal:close'; + $.modal.AFTER_CLOSE = 'modal:after-close'; + $.modal.AJAX_SEND = 'modal:ajax:send'; + $.modal.AJAX_SUCCESS = 'modal:ajax:success'; + $.modal.AJAX_FAIL = 'modal:ajax:fail'; + $.modal.AJAX_COMPLETE = 'modal:ajax:complete'; + + $.fn.modal = function(options){ + if (this.length === 1) { + new $.modal(this, options); + } + return this; + }; + + // Automatically bind links with rel="modal:close" to, well, close the modal. + $(document).on('click.modal', 'a[rel="modal:close"]', $.modal.close); + $(document).on('click.modal', 'a[rel="modal:open"]', function(event) { + event.preventDefault(); + $(this).modal(); + }); +})(jQuery); diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 9dccef1400..904a687ad4 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -30,6 +30,7 @@ type Hive struct { addr kademlia.Address kad *kademlia.Kademlia path string + quit chan bool toggle chan bool more chan bool @@ -106,6 +107,7 @@ func (self *Hive) Addr() kademlia.Address { func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) { self.toggle = make(chan bool) self.more = make(chan bool) + self.quit = make(chan bool) self.id = id self.listenAddr = listenAddr err = self.kad.Load(self.path, nil) @@ -123,13 +125,15 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee // to attempt to write to more (remove Peer when shutting down) return } - node, proxLimit := self.kad.FindBest() + node, need, proxLimit := self.kad.Suggest() + if node != nil && len(node.Url) > 0 { - glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: call for bee %v", node.Url) + glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url) // enode or any lower level connection address is unnecessary in future // discovery table is used to look it up. connectPeer(node.Url) - } else if proxLimit > -1 { + } + if need { // a random peer is taken from the table peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1) if len(peers) > 0 { @@ -138,15 +142,21 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee req := &retrieveRequestMsgData{ Key: storage.Key(randAddr[:]), } - glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee in area %v messenger bee %v", randAddr, peers[0]) + glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0]) peers[0].(*peer).retrieve(req) + } else { + glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer") } - self.toggle <- true glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive") } else { - self.toggle <- false + glog.V(logger.Info).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees") } - glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()) + select { + case self.toggle <- need: + case <-self.quit: + return + } + glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()) } }() return @@ -165,14 +175,11 @@ func (self *Hive) keepAlive() { if self.kad.DBCount() > 0 { select { case self.more <- true: + glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz wakeup") default: } } - case need, alive := <-self.toggle: - if !alive { - self.more <- false - return - } + case need := <-self.toggle: if alarm == nil && need { alarm = time.NewTicker(time.Duration(self.callInterval)).C } @@ -180,20 +187,31 @@ func (self *Hive) keepAlive() { alarm = nil } + case <-self.quit: + return } } } func (self *Hive) Stop() error { // closing toggle channel quits the updateloop - close(self.toggle) + close(self.quit) return self.kad.Save(self.path, saveSync) } // called at the end of a successful protocol handshake -func (self *Hive) addPeer(p *peer) { +func (self *Hive) addPeer(p *peer) error { + defer func() { + select { + case self.more <- true: + default: + } + }() glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: hi new bee %v", p) - self.kad.On(p, loadSync) + err := self.kad.On(p, loadSync) + if err != nil { + return err + } // self lookup (can be encoded as nil/zero key since peers addr known) + no id () // the most common way of saying hi in bzz is initiation of gossip // let me know about anyone new from my hood , here is the storageradius @@ -201,10 +219,8 @@ func (self *Hive) addPeer(p *peer) { // we do not record as request or forward it, just reply with peers p.retrieve(&retrieveRequestMsgData{}) glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: 'whatsup wheresdaparty' sent to %v", p) - select { - case self.more <- true: - default: - } + + return nil } // called after peer disconnected @@ -241,7 +257,7 @@ func (self *Hive) DropAll() { // contructor for kademlia.NodeRecord based on peer address alone // TODO: should go away and only addr passed to kademlia func newNodeRecord(addr *peerAddr) *kademlia.NodeRecord { - now := kademlia.Time(time.Now()) + now := time.Now() return &kademlia.NodeRecord{ Addr: addr.Addr, Url: addr.String(), @@ -336,7 +352,7 @@ func (self *Hive) peers(req *retrieveRequestMsgData) { for _, peer := range self.getPeers(key, int(req.MaxPeers)) { addrs = append(addrs, peer.remoteAddr) } - glog.V(logger.Detail).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %x", len(addrs), req.from, req.Id, req.Key.Log()) + glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log()) peersData := &peersMsgData{ Peers: addrs, diff --git a/swarm/network/kademlia/kaddb.go b/swarm/network/kademlia/kaddb.go index d7c6a5be13..33cae221cb 100644 --- a/swarm/network/kademlia/kaddb.go +++ b/swarm/network/kademlia/kaddb.go @@ -12,26 +12,6 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" ) -type Time time.Time - -func (t *Time) MarshalJSON() (out []byte, err error) { - return []byte(fmt.Sprintf("%d", t.Unix())), nil -} - -func (t *Time) UnmarshalJSON(value []byte) error { - var i int64 - _, err := fmt.Sscanf(string(value), "%d", &i) - if err != nil { - return err - } - *t = Time(time.Unix(i, 0)) - return nil -} - -func (t Time) Unix() int64 { - return time.Time(t).Unix() -} - type NodeData interface { json.Marshaler json.Unmarshaler @@ -41,17 +21,17 @@ type NodeData interface { type NodeRecord struct { Addr Address // address of node Url string // Url, used to connect to node - After Time // next call after time - Seen Time // last connected at time + After time.Time // next call after time + Seen time.Time // last connected at time Meta *json.RawMessage // arbitrary metadata saved for a peer - node Node - connected bool + node Node } -// set checked to current time, func (self *NodeRecord) setSeen() { - self.Seen = Time(time.Now()) + t := time.Now() + self.Seen = t + self.After = t } func (self *NodeRecord) String() string { @@ -64,7 +44,7 @@ type KadDb struct { Nodes [][]*NodeRecord index map[Address]*NodeRecord cursors []int - lock sync.Mutex + lock sync.RWMutex purgeInterval time.Duration initialRetryInterval time.Duration connRetryExp int @@ -142,16 +122,19 @@ This is used to pick candidates for live nodes that are most wanted for a higly connected low centrality network structure for Swarm which best suits for a Kademlia-style routing. -The candidate is chosen using the following strategy. +* Starting as naive node with empty db, this implements Kademlia bootstrapping +* As a mature node, it fills short lines. All on demand. + +The candidate is chosen using the following strategy: We check for missing online nodes in the buckets for 1 upto Max BucketSize rounds. On each round we proceed from the low to high proximity order buckets. If the number of active nodes (=connected peers) is < rounds, then start looking for a known candidate. To determine if there is a candidate to recommend the -node record database row corresponding to the bucket is checked. +kaddb node record database row corresponding to the bucket is checked. If the row cursor is on position i, the ith element in the row is chosen. If the record is scheduled not to be retried before NOW, the next element is taken. -If the record is scheduled can be retried, it is set as checked, scheduled for +If the record is scheduled to be retried, it is set as checked, scheduled for checking and is returned. The time of the next check is in X (duration) such that X = ConnRetryExp * delta where delta is the time past since the last check and ConnRetryExp is constant obsoletion factor. (Note that when node records are added @@ -167,121 +150,109 @@ offline past peer) || (proxBin(a) < proxBin(b) && |proxBin(a)| == |proxBin(b)|) || (proxBin(a) == proxBin(b) && lastChecked(a) < lastChecked(b)) -This has double role. Starting as naive node with empty db, this implements -Kademlia bootstrapping -As a mature node, it fills short lines. All on demand. The second argument returned names the first missing slot found */ -func (self *KadDb) findBest(bucketSize int, binsize func(int) int) (node *NodeRecord, proxLimit int) { - // return value -1 indicates that buckets are filled in all - proxLimit = -1 +func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRecord, need bool, proxLimit int) { + // return nil, proxLimit indicates that all buckets are filled defer self.lock.Unlock() self.lock.Lock() - var interval int64 + var interval time.Duration var found bool - for rounds := 1; rounds <= bucketSize; rounds++ { + var purge []bool + var delta time.Duration + var cursor int + var count int + var after time.Time + + // iterate over columns maximum bucketsize times + for rounds := 1; rounds <= maxBinSize; rounds++ { ROUND: + // iterate over rows from PO 0 upto MaxProx for po, dbrow := range self.Nodes { - if po > len(self.Nodes) { - break ROUND + // if row has rounds connected peers, then take the next + if binSize(po) >= rounds { + continue ROUND } - size := binsize(po) - if size < rounds { - if proxLimit < 0 { - // set the first missing slot found - proxLimit = po + if !need { + // set proxlimit to the PO where the first missing slot is found + proxLimit = po + need = true + } + purge = make([]bool, len(dbrow)) + + // there is a missing slot - finding a node to connect to + // select a node record from the relavant kaddb row (of identical prox order) + ROW: + for cursor = self.cursors[po]; !found && count < len(dbrow); cursor = (cursor + 1) % len(dbrow) { + count++ + node = dbrow[cursor] + + // skip already connected nodes + if node.node != nil { + glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d/%d) already connected", node.Addr, po, cursor, len(dbrow)) + continue ROW } - var count int - var purge []int - n := self.cursors[po] - // try node records in the relavant kaddb row (of identical prox order) - // if they are ripe for checking - ROW: - for count < len(dbrow) { - node = dbrow[n] - - // skip already connected nodes - if !node.connected { - - glog.V(logger.Detail).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) not to be retried before %v", node.Addr, po, n, node.After) - - // time since last known connection attempt - delta := node.After.Unix() - node.Seen.Unix() - // if delta < 4 { - // node.After = Time(time.Time{}) - // } - - // if node is scheduled to connect - if time.Time(node.After).Before(time.Now()) { - - // if checked longer than purge interval - if time.Time(node.Seen).Add(self.purgeInterval).Before(time.Now()) { - // delete node - purge = append(purge, n) - glog.V(logger.Detail).Infof("[KΛÐ]: inactive node record %v (PO%03d:%d) last check: %v, next check: %v", node.Addr, po, n, node.Seen, node.After) - } else { - // scheduling next check - if (node.After == Time(time.Time{})) { - node.After = Time(time.Now().Add(self.initialRetryInterval)) - } else { - interval = delta * int64(self.connRetryExp) - node.After = Time(time.Unix(time.Now().Unix()+interval, 0)) - } - - glog.V(logger.Detail).Infof("[KΛÐ]: serve node record %v (PO%03d:%d), last check: %v, next check: %v", node.Addr, po, n, node.Seen, node.After) - } - found = true - break ROW - } - glog.V(logger.Detail).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) not ready. skipped. not to be retried before: %v", node.Addr, po, n, node.After) - } // if node.node == nil - n++ - count++ - // cycle: n = n % len(dbrow) - if n >= len(dbrow) { - n = 0 - } + // if node is scheduled to connect + if time.Time(node.After).After(time.Now()) { + glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) skipped. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After) + continue ROW } - self.cursors[po] = n - self.delete(po, purge...) - if found { - glog.V(logger.Detail).Infof("[KΛÐ]: rounds %d: prox limit: PO%03d\n%v", rounds, proxLimit, node) - node.setSeen() - return - } - } // if len < rounds - } // for po-s - glog.V(logger.Detail).Infof("[KΛÐ]: rounds %d: proxlimit: PO%03d", rounds, proxLimit) - if proxLimit == 0 || proxLimit < 0 && bucketSize == rounds { - return - } - } // for round - return + delta = time.Since(time.Time(node.Seen)) + if delta < self.initialRetryInterval { + delta = self.initialRetryInterval + } + if delta > self.purgeInterval { + // remove node + purge[cursor] = true + glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen) + continue ROW + } + + glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) ready to be tried. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After) + + // scheduling next check + interval = time.Duration(delta * time.Duration(self.connRetryExp)) + after = time.Now().Add(interval) + + glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval) + node.After = after + found = true + } // ROW + self.cursors[po] = cursor + self.delete(po, purge) + if found { + return node, need, proxLimit + } + } // ROUND + } // ROUNDS + + return nil, need, proxLimit } // deletes the noderecords of a kaddb row corresponding to the indexes // caller must hold the dblock // the call is unsafe, no index checks -func (self *KadDb) delete(row int, indexes ...int) { - var prev int +func (self *KadDb) delete(row int, purge []bool) { var nodes []*NodeRecord dbrow := self.Nodes[row] - for _, next := range indexes { - // need to adjust dbcursor - if next > 0 { - if next <= self.cursors[row] { - self.cursors[row]-- - } - nodes = append(nodes, dbrow[prev:next]...) + for i, del := range purge { + if i == self.cursors[row] { + //reset cursor + self.cursors[row] = len(nodes) } - prev = next + 1 - delete(self.index, dbrow[next].Addr) + // delete the entry to be purged + if del { + delete(self.index, dbrow[i].Addr) + continue + } + // otherwise append to new list + nodes = append(nodes, dbrow[i]) } - self.Nodes[row] = append(nodes, dbrow[prev:]...) + self.Nodes[row] = nodes } // save persists kaddb on disk (written to file on path in json format. @@ -294,8 +265,8 @@ func (self *KadDb) save(path string, cb func(*NodeRecord, Node)) error { for _, b := range self.Nodes { for _, node := range b { n++ - node.After = Time(time.Now()) - node.Seen = Time(time.Now()) + node.After = time.Now() + node.Seen = time.Now() if cb != nil { cb(node, node.node) } @@ -331,24 +302,25 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro return } var n int - var purge []int + var purge []bool for po, b := range self.Nodes { + purge = make([]bool, len(b)) ROW: for i, node := range b { if cb != nil { err = cb(node, node.node) if err != nil { - purge = append(purge, i) + purge[i] = true continue ROW } } n++ - if (node.After == Time(time.Time{})) { - node.After = Time(time.Now()) + if (node.After == time.Time{}) { + node.After = time.Now() } self.index[node.Addr] = node } - self.delete(po, purge...) + self.delete(po, purge) } glog.V(logger.Info).Infof("[KΛÐ] loaded kaddb with %v nodes from %v", n, path) diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 602db1445d..8fcfdc08e1 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -12,16 +12,17 @@ import ( ) const ( - bucketSize = 3 - proxBinSize = 4 + bucketSize = 4 + proxBinSize = 2 maxProx = 8 connRetryExp = 2 + maxPeers = 100 ) var ( purgeInterval = 42 * time.Hour - initialRetryInterval = 42 * 100 * time.Millisecond - maxIdleInterval = 42 * 10 * time.Second + initialRetryInterval = 42 * time.Millisecond + maxIdleInterval = 42 * 100 * time.Millisecond ) type KadParams struct { @@ -31,6 +32,7 @@ type KadParams struct { BucketSize int PurgeInterval time.Duration InitialRetryInterval time.Duration + MaxIdleInterval time.Duration ConnRetryExp int } @@ -41,6 +43,7 @@ func NewKadParams() *KadParams { BucketSize: bucketSize, PurgeInterval: purgeInterval, InitialRetryInterval: initialRetryInterval, + MaxIdleInterval: maxIdleInterval, ConnRetryExp: connRetryExp, } } @@ -52,7 +55,7 @@ type Kademlia struct { proxLimit int // state, the PO of the first row of the most proximate bin proxSize int // state, the number of peers in the most proximate bin count int // number of active peers (w live connection) - buckets []*bucket // the actual bins + buckets [][]Node // the actual bins db *KadDb // kaddb, node record database lock sync.RWMutex // mutex to access buckets } @@ -68,14 +71,7 @@ type Node interface { // add is the base address of the table // params is KadParams configuration func New(addr Address, params *KadParams) *Kademlia { - buckets := make([]*bucket, params.MaxProx+1) - for i, _ := range buckets { - buckets[i] = &bucket{size: params.BucketSize} // will initialise bucket{int(0),[]Node(nil),sync.Mutex} - } - glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr) - - // ! temporary hack fixme: - params.ProxBinSize = 4 + buckets := make([][]Node, params.MaxProx+1) return &Kademlia{ addr: addr, KadParams: params, @@ -104,14 +100,12 @@ func (self *Kademlia) DBCount() int { // On is the entry point called when a new nodes is added // unsafe in that node is not checked to be already active node (to be called once) func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) { + glog.V(logger.Warn).Infof("[KΛÐ]: %v", self) defer self.lock.Unlock() self.lock.Lock() index := self.proximityBin(node.Addr()) record := self.db.findOrCreate(index, node.Addr(), node.Url()) - // callback on add node - // setting the node on the record, set it checked (for connectivity) - record.node = node if cb != nil { err = cb(record, node) @@ -119,33 +113,46 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error if err != nil { return fmt.Errorf("unable to add node %v, callback error: %v", node.Addr(), err) } - glog.V(logger.Info).Infof("[KΛÐ]: add node record %v with node %v", record, node) + glog.V(logger.Debug).Infof("[KΛÐ]: add node record %v with node %v", record, node) } - record.connected = true // insert in kademlia table of active nodes bucket := self.buckets[index] // if bucket is full insertion replaces the worst node // TODO: give priority to peers with active traffic - replaced, err := bucket.insert(node) - if err != nil { - glog.V(logger.Debug).Infof("[KΛÐ]: node %v not needed: %v", node, err) - return err - // no prox adjustment needed - // do not change count + if len(bucket) >= self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation + // always rotate peers + idle := self.MaxIdleInterval + var pos int + var replaced Node + for i, p := range bucket { + idleInt := time.Since(p.LastActive()) + if idleInt > idle { + idle = idleInt + pos = i + replaced = p + } + } + if replaced == nil { + glog.V(logger.Debug).Infof("[KΛÐ]: all peers wanted, PO%03d bucket full", index) + return fmt.Errorf("bucket full") + } + glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v (idle for %v > %v)", replaced, node, idle, self.MaxIdleInterval) + replaced.Drop() + self.buckets[index] = append(bucket[:pos], bucket[(pos+1):]...) + // there is no change in bucket cardinalities so no prox limit adjustment is needed + return nil + } else { + self.buckets[index] = append(bucket, node) + glog.V(logger.Debug).Infof("[KΛÐ]: add node %v to table", node) + self.count++ + self.setProxLimit(index, true) } - if replaced != nil { - glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v ", replaced, node) - return - } - // new node added - glog.V(logger.Info).Infof("[KΛÐ]: add node %v to table", node) - self.count++ - self.setProxLimit(index, false) - return + record.node = node + return nil } -// is the entrypoint called when a node is taken offline +// Off is the called when a node is taken offline (from the protocol main loop exit) func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { self.lock.Lock() defer self.lock.Unlock() @@ -153,70 +160,73 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { var found bool index := self.proximityBin(node.Addr()) bucket := self.buckets[index] - for i := 0; i < len(bucket.nodes); i++ { - if node.Addr() == bucket.nodes[i].Addr() { + for i := 0; i < len(bucket); i++ { + if node.Addr() == bucket[i].Addr() { found = true - bucket.nodes = append(bucket.nodes[:i], bucket.nodes[(i+1):]...) + self.buckets[index] = append(bucket[:i], bucket[(i+1):]...) + break } } if !found { - return + // gracefully return without error if peer already unregistered + glog.V(logger.Warn).Infof("[KΛÐ]: remove node %v not in table, population now is %v", node, self.count) + return nil } - glog.V(logger.Info).Infof("[KΛÐ]: remove node %v from table", node) self.count-- - if len(bucket.nodes) < bucket.size { - err = fmt.Errorf("insufficient nodes (%v) in bucket %v", len(bucket.nodes), index) - } + glog.V(logger.Debug).Infof("[KΛÐ]: remove node %v from table, population now is %v", node, self.count) + self.setProxLimit(index, false) - self.setProxLimit(index, true) - - r := self.db.index[node.Addr()] + record := self.db.index[node.Addr()] // callback on remove if cb != nil { - cb(r, r.node) + cb(record, record.node) } - r.node = nil - r.connected = false + record.node = nil return } // proxLimit is dynamically adjusted so that // 1) there is no empty buckets in bin < proxLimit and -// 2) the sum of all items sare the maximpossible but lower than ProxBinSize +// 2) the sum of all items are the minimum possible but higher than ProxBinSize // adjust Prox (proxLimit and proxSize after an insertion/removal of nodes) // caller holds the lock -func (self *Kademlia) setProxLimit(r int, off bool) { - // glog.V(logger.Info).Infof("[KΛÐ]: adjust proxbin for (bin: %v, off: %v)", r, off) - if r < self.proxLimit && len(self.buckets[r].nodes) > 0 { +func (self *Kademlia) setProxLimit(r int, on bool) { + // if the change is outside the core (PO lower) + // and the change does not leave a bucket empty then + // no adjustment needed + if r < self.proxLimit && len(self.buckets[r]) > 0 { return } - glog.V(logger.Detail).Infof("[KΛÐ]: set proxbin (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) - if off { - self.proxSize-- - for (self.proxSize < self.ProxBinSize || r < self.proxLimit) && - self.proxLimit > 0 { - // - self.proxLimit-- - self.proxSize += len(self.buckets[self.proxLimit].nodes) - glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) - } - glog.V(logger.Detail).Infof("%v", self) - return - } - self.proxSize++ - for self.proxLimit < self.MaxProx && - len(self.buckets[self.proxLimit].nodes) > 0 && - self.proxSize-len(self.buckets[self.proxLimit].nodes) >= self.ProxBinSize { - // - self.proxSize -= len(self.buckets[self.proxLimit].nodes) - self.proxLimit++ - glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) - } - glog.V(logger.Detail).Infof("%v", self) + // if on=a node was added, then r must be within prox limit so increment cardinality + if on { + self.proxSize++ + curr := len(self.buckets[self.proxLimit]) + // if now core is big enough without the furthest bucket, then contract + // this can result in more than one bucket change + for self.proxSize >= self.ProxBinSize+curr && curr > 0 { + self.proxSize -= curr + self.proxLimit++ + curr = len(self.buckets[self.proxLimit]) + glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r) + } + return + } + // otherwise + if r >= self.proxLimit { + self.proxSize-- + } + // expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality + for (self.proxSize < self.ProxBinSize || r < self.proxLimit) && + self.proxLimit > 0 { + // + self.proxLimit-- + self.proxSize += len(self.buckets[self.proxLimit]) + glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r) + } } /* @@ -225,62 +235,54 @@ as the target. The most proximate bin will be the union of the bins between proxLimit and MaxProx. */ func (self *Kademlia) FindClosest(target Address, max int) []Node { - defer self.lock.RUnlock() - self.lock.RLock() + self.lock.Lock() + defer self.lock.Unlock() + r := nodesByDistance{ target: target, } - index := self.proximityBin(target) - start := index - var down bool - if index >= self.proxLimit { - index = self.proxLimit - start = self.MaxProx - down = true - } - var n int + po := self.proximityBin(target) + index := po + step := 1 + glog.V(logger.Detail).Infof("[KΛÐ]: serving %v nodes at %v (PO%02d)", max, index, po) + + // if max is set to 0, just want a full bucket, dynamic number + min := max + // set limit to max limit := max if max == 0 { - limit = 1000 + min = 1 + limit = maxPeers } - for { - bucket := self.buckets[start].nodes - for i := 0; i < len(bucket); i++ { - r.push(bucket[i], limit) + var n int + for index >= 0 { + // add entire bucket + for _, p := range self.buckets[index] { + r.push(p, limit) n++ } - if max == 0 && start <= index && (n > 0 || start == 0) || max > 0 && down && start <= index && (n >= limit || n == self.count || start == 0) { + // terminate if index reached the bottom or enough peers > min + glog.V(logger.Detail).Infof("[KΛÐ]: add %v -> %v (PO%02d, PO%03d)", len(self.buckets[index]), n, index, po) + if n >= min && (step < 0 || max == 0) { break } - if down { - start-- - } else { - if start == self.MaxProx { - if index == 0 { - break - } - start = index - 1 - down = true - } else { - start++ - } + // reach top most non-empty PO bucket, turn around + if index == self.MaxProx { + index = po + step = -1 } + index += step } - glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (=<%d) nodes for target lookup %v (PO%d)", n, self.MaxProx, target, index) + glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (<=%d) nodes for target lookup %v (PO%03d)", n, max, target, po) return r.nodes } -func (self *Kademlia) binsize(p int) int { - b := self.buckets[p] - defer b.lock.RUnlock() - b.lock.RLock() - return len(b.nodes) -} - -func (self *Kademlia) FindBest() (node *NodeRecord, proxLimit int) { - return self.db.findBest(self.BucketSize, self.binsize) +func (self *Kademlia) Suggest() (*NodeRecord, bool, int) { + defer self.lock.RUnlock() + self.lock.RLock() + return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) }) } // adds node records to kaddb (persisted node record db) @@ -288,13 +290,6 @@ func (self *Kademlia) Add(nrs []*NodeRecord) { self.db.add(nrs, self.proximityBin) } -// in situ mutable bucket -type bucket struct { - size int - nodes []Node - lock sync.RWMutex -} - // nodesByDistance is a list of nodes, ordered by distance to target. type nodesByDistance struct { nodes []Node @@ -331,27 +326,6 @@ func (h *nodesByDistance) push(node Node, max int) { } } -// insert adds a peer to a bucket either by appending to existing items if -// bucket length does not exceed bucketSize, or by replacing the worst -// Node in the bucket -func (self *bucket) insert(node Node) (replaced Node, err error) { - self.lock.Lock() - defer self.lock.Unlock() - if len(self.nodes) >= self.size { // >= allows us to add peers beyond the bucketsize limitation - // dev p2p kicks out nodes idle for > 30 s, so here we never replace nodes if - // bucket is full - return nil, fmt.Errorf("bucket full") - } - self.nodes = append(self.nodes, node) - return -} - -func (self *bucket) length(node Node) int { - self.lock.Lock() - defer self.lock.Unlock() - return len(self.nodes) -} - /* Taking the proximity order relative to a fix point x classifies the points in the space (n byte long byte sequences) into bins. Items in each are at @@ -396,43 +370,46 @@ func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err e } // kademlia table + kaddb table displayed with ascii -// callerholds the lock func (self *Kademlia) String() string { + defer self.lock.RUnlock() + self.lock.RLock() + defer self.db.lock.RUnlock() + self.db.lock.RLock() var rows []string rows = append(rows, "=========================================================================") - rows = append(rows, fmt.Sprintf("KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.Count(), self.DBCount())) - rows = append(rows, fmt.Sprintf("%v : MaxProx: %d, ProxBinSize: %d, BucketSize: %d, proxLimit: %d, proxSize: %d", time.Now(), self.MaxProx, self.ProxBinSize, self.BucketSize, self.proxLimit, self.proxSize)) + rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6])) + rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize)) + rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize)) - for i, b := range self.buckets { + for i, bucket := range self.buckets { if i == self.proxLimit { - rows = append(rows, fmt.Sprintf("===================== PROX LIMIT: %d =================================", i)) + rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i)) } - row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(b.nodes))} + row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))} var k int c := self.db.cursors[i] - for ; k < len(b.nodes); k++ { - p := b.nodes[(c+k)%len(b.nodes)] - row = append(row, fmt.Sprintf("%s", p.Addr().String()[:8])) - if k == 3 { + for ; k < len(bucket); k++ { + p := bucket[(c+k)%len(bucket)] + row = append(row, p.Addr().String()[:6]) + if k == 4 { break } } - for ; k < 3; k++ { - row = append(row, " ") + for ; k < 4; k++ { + row = append(row, " ") } row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i])) for j, p := range self.db.Nodes[i] { - row = append(row, fmt.Sprintf("%08x", p.Addr[:4])) - if j == 2 { + row = append(row, p.Addr.String()[:6]) + if j == 3 { break } } rows = append(rows, strings.Join(row, " ")) if i == self.MaxProx { - break } } rows = append(rows, "=========================================================================") diff --git a/swarm/network/kademlia/kademlia_test.go b/swarm/network/kademlia/kademlia_test.go index f36158c378..0e162e99b8 100644 --- a/swarm/network/kademlia/kademlia_test.go +++ b/swarm/network/kademlia/kademlia_test.go @@ -2,6 +2,7 @@ package kademlia import ( "fmt" + "math" "math/rand" "reflect" "testing" @@ -11,8 +12,8 @@ import ( var ( quickrand = rand.New(rand.NewSource(time.Now().Unix())) - quickcfgFindClosest = &quick.Config{MaxCount: 5000, Rand: quickrand} - quickcfgBootStrap = &quick.Config{MaxCount: 1000, Rand: quickrand} + quickcfgFindClosest = &quick.Config{MaxCount: 50, Rand: quickrand} + quickcfgBootStrap = &quick.Config{MaxCount: 100, Rand: quickrand} ) type testNode struct { @@ -60,43 +61,36 @@ func TestBootstrap(t *testing.T) { kad := New(test.Self, params) var err error - addr := RandomAddress() - prox := proximity(addr, test.Self) - - for p := 0; p <= prox; p++ { + for p := 0; p < 9; p++ { var nrs []*NodeRecord - for i := 0; i < 3; i++ { + n := math.Pow(float64(2), float64(7-p)) + for i := 0; i < int(n); i++ { + addr := RandomAddressAt(test.Self, p) nrs = append(nrs, &NodeRecord{ - Addr: RandomAddressAt(test.Self, p), + Addr: addr, }) } kad.Add(nrs) } - node := &testNode{addr} + node := &testNode{test.Self} n := 0 for n < 100 { err = kad.On(node, nil) if err != nil { - t.Errorf("backend not accepting node") - return false + t.Fatalf("backend not accepting node: %v", err) } - var nrs []*NodeRecord - prox := proximity(test.Self, node.addr) - for i := 0; i < 13; i++ { - nrs = append(nrs, &NodeRecord{ - Addr: RandomAddressAt(test.Self, prox+1), - }) - } - kad.Add(nrs) - record, _ := kad.FindBest() - if record == nil { + record, need, _ := kad.Suggest() + if !need { break } - node = &testNode{record.Addr} n++ + if record == nil { + continue + } + node = &testNode{record.Addr} } exp := test.BucketSize * (test.MaxProx + 1) if kad.Count() != exp { @@ -116,15 +110,13 @@ func TestFindClosest(t *testing.T) { test := func(test *FindClosestTest) bool { // for any node kad.le, Target and N params := NewKadParams() - params.MaxProx = 10 + params.MaxProx = 7 kad := New(test.Self, params) var err error - // t.Logf("FindClosestTest %v: %v\n", len(test.All), test) for _, node := range test.All { err = kad.On(node, nil) - if err != nil { - t.Errorf("backend not accepting node") - return false + if err != nil && err.Error() != "bucket full" { + t.Fatalf("backend not accepting node: %v", err) } } @@ -157,17 +149,13 @@ func TestFindClosest(t *testing.T) { // check that the result nodes have minimum distance to target. farthestResult := nodes[len(nodes)-1].Addr() for i, b := range kad.buckets { - for j, n := range b.nodes { + for j, n := range b { if contains(nodes, n.Addr()) { continue // don't run the check below for nodes in result } if test.Target.ProxCmp(n.Addr(), farthestResult) < 0 { _ = i * j t.Errorf("kad.le contains node that is closer to target but it's not in result") - // t.Logf("bucket %v, item %v\n", i, j) - // t.Logf(" Target: %x", test.Target) - // t.Logf(" Farthest Result: %x", farthestResult) - // t.Logf(" ID: %x (%d)", n.Addr(), kad.proximityBin(n.Addr())) return false } } @@ -193,7 +181,7 @@ func TestProxAdjust(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) self := gen(Address{}, r).(Address) params := NewKadParams() - params.MaxProx = 10 + params.MaxProx = 7 kad := New(self, params) var err error @@ -201,15 +189,13 @@ func TestProxAdjust(t *testing.T) { a := gen(Address{}, r).(Address) addresses = append(addresses, a) err = kad.On(&testNode{addr: a}, nil) - if err != nil { - t.Errorf("backend not accepting node") - return + if err != nil && err.Error() != "bucket full" { + t.Fatalf("backend not accepting node: %v", err) } if !kad.proxCheck(t) { return } } - test := func(test *proxTest) bool { node := &testNode{test.addr} if test.add { @@ -229,35 +215,33 @@ func TestSaveLoad(t *testing.T) { addresses := gen([]Address{}, r).([]Address) self := RandomAddress() params := NewKadParams() - params.MaxProx = 10 + params.MaxProx = 7 kad := New(self, params) var err error for _, a := range addresses { err = kad.On(&testNode{addr: a}, nil) - if err != nil { - t.Errorf("backend not accepting node") - return + if err != nil && err.Error() != "bucket full" { + t.Fatalf("backend not accepting node: %v", err) } } nodes := kad.FindClosest(self, 100) path := "/tmp/bzz.peers" err = kad.Save(path, nil) - if err != nil { + if err != nil && err.Error() != "bucket full" { t.Fatalf("unepected error saving kaddb: %v", err) } kad = New(self, params) err = kad.Load(path, nil) - if err != nil { + if err != nil && err.Error() != "bucket full" { t.Fatalf("unepected error loading kaddb: %v", err) } for _, b := range kad.db.Nodes { for _, node := range b { err = kad.On(&testNode{node.Addr}, nil) - if err != nil { - t.Errorf("backend not accepting node") - return + if err != nil && err.Error() != "bucket full" { + t.Fatalf("backend not accepting node: %v", err) } } } @@ -270,30 +254,30 @@ func TestSaveLoad(t *testing.T) { } func (self *Kademlia) proxCheck(t *testing.T) bool { - var sum, i int - var b *bucket - for i, b = range self.buckets { - l := len(b.nodes) + var sum int + for i, b := range self.buckets { + l := len(b) // if we are in the high prox multibucket if i >= self.proxLimit { sum += l } else if l == 0 { - t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b.nodes), self.proxLimit, self) + t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self) return false } } // check if merged high prox bucket does not exceed size if sum > 0 { - // if sum > self.ProxBinSize { - // t.Errorf("bucket %d is empty, yet proxSize is %d\n%v", i, self.proxSize, self) - // return false - // } if sum != self.proxSize { t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize) return false } - if self.proxLimit > 0 && sum+len(self.buckets[self.proxLimit-1].nodes) < self.ProxBinSize { - t.Errorf("proxBinSize incorrect, expected %v got %v", sum, self.proxSize) + last := len(self.buckets[self.proxLimit]) + if last > 0 && sum >= self.ProxBinSize+last { + t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)\n%v", self.proxLimit, last, sum-last, self.ProxBinSize, self) + return false + } + if self.proxLimit > 0 && sum < self.ProxBinSize { + t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize) return false } } @@ -309,7 +293,7 @@ type bootstrapTest struct { func (*bootstrapTest) Generate(rand *rand.Rand, size int) reflect.Value { t := &bootstrapTest{ Self: gen(Address{}, rand).(Address), - MaxProx: 10 + rand.Intn(3), + MaxProx: 5 + rand.Intn(2), BucketSize: rand.Intn(3) + 1, } return reflect.ValueOf(t) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 6f27f68b26..7ba7755e34 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -49,6 +49,7 @@ const ( ErrExtraStatusMsg ErrSwap ErrSync + ErrUnwanted ) var errorToString = map[int]string{ @@ -61,6 +62,7 @@ var errorToString = map[int]string{ ErrExtraStatusMsg: "Extra status message", ErrSwap: "SWAP error", ErrSync: "Sync error", + ErrUnwanted: "Unwanted peer", } // bzz represents the swarm wire protocol @@ -179,7 +181,8 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend bind.Backe // the main forever loop that handles incoming requests for { if self.hive.blockRead { - time.Sleep(1 * time.Second) + glog.V(logger.Warn).Infof("[BZZ] Cannot read network") + time.Sleep(100 * time.Millisecond) continue } err = self.handle() @@ -223,7 +226,7 @@ func (self *bzz) handle() error { if err := msg.Decode(&req); err != nil { return self.protoError(ErrDecode, "<- %v: %v", msg, err) } - glog.V(logger.Debug).Infof("[BZZ] incoming store request: %s", req.String()) + glog.V(logger.Detail).Infof("[BZZ] incoming store request: %s", req.String()) // swap accounting is done within forwarding self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self}) @@ -254,7 +257,7 @@ func (self *bzz) handle() error { return self.protoError(ErrDecode, "<- %v: %v", msg, err) } req.from = &peer{bzz: self} - glog.V(logger.Debug).Infof("[BZZ] <- peer addresses: %v", req) + glog.V(logger.Detail).Infof("[BZZ] <- peer addresses: %v", req) self.hive.HandlePeersMsg(&req, &peer{bzz: self}) case syncRequestMsg: @@ -366,7 +369,10 @@ func (self *bzz) handleStatus() (err error) { } glog.V(logger.Info).Infof("[BZZ] Peer %08x is [bzz] capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId) - self.hive.addPeer(&peer{bzz: self}) + err = self.hive.addPeer(&peer{bzz: self}) + if err != nil { + return self.protoError(ErrUnwanted, "%v", err) + } // hive sets syncstate so sync should start after node added glog.V(logger.Info).Infof("[BZZ] syncronisation request sent with %v", self.syncState) @@ -516,7 +522,6 @@ func (self *bzz) send(msg uint64, data interface{}) error { if self.hive.blockWrite { return fmt.Errorf("network write blocked") } - // self.messages = append(self.messages, "") glog.V(logger.Detail).Infof("[BZZ] -> %v: %v (%T) to %v", msg, data, data, self) err := p2p.Send(self.rw, msg, data) if err != nil { diff --git a/swarm/network/syncdb.go b/swarm/network/syncdb.go index f91ed9a50d..aabc2d9b03 100644 --- a/swarm/network/syncdb.go +++ b/swarm/network/syncdb.go @@ -126,7 +126,7 @@ LOOP: // if syncdb is stopped. In this case we need to save the item to the db more = deliver(req, self.quit) if !more { - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] quit: switching to db. session tally (db/total): %v/%v", self.priority, self.dbTotal, self.total) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total) // received quit signal, save request currently waiting delivery // by switching to db mode and closing the buffer buffer = nil @@ -136,12 +136,12 @@ LOOP: break // break from select, this item will be written to the db } self.total++ - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] deliver (db/total): %v/%v", self.priority, self.dbTotal, self.total) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total) // by the time deliver returns, there were new writes to the buffer // if buffer contention is detected, switch to db mode which drains // the buffer so no process will block on pushing store requests if len(buffer) == cap(buffer) { - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.priority, cap(buffer), self.dbTotal, self.total) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total) buffer = nil db = self.buffer } @@ -154,18 +154,18 @@ LOOP: binary.BigEndian.PutUint64(counterValue, counter) batch.Put(self.counterKey, counterValue) // persist counter in batch self.writeSyncBatch(batch) // save batch - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save current batch to db", self.priority) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority) break LOOP } self.dbTotal++ self.total++ - // otherwise break after selec + // otherwise break after select case dbSize = <-self.batch: // explicit request for batch if inBatch == 0 && quit != nil { // there was no writes since the last batch so db depleted // switch to buffer mode - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] empty db: switching to buffer", self.priority) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority) db = nil buffer = self.buffer dbSize <- 0 // indicates to 'caller' that batch has been written @@ -174,7 +174,7 @@ LOOP: } binary.BigEndian.PutUint64(counterValue, counter) batch.Put(self.counterKey, counterValue) - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] write batch %v/%v - %x - %x", self.priority, inBatch, counter, self.counterKey, counterValue) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue) batch = self.writeSyncBatch(batch) dbSize <- inBatch // indicates to 'caller' that batch has been written inBatch = 0 @@ -186,7 +186,7 @@ LOOP: db = self.buffer buffer = nil quit = nil - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save buffer to db", self.priority) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority) close(db) continue LOOP } @@ -194,15 +194,15 @@ LOOP: // only get here if we put req into db entry, err = self.newSyncDbEntry(req, counter) if err != nil { - glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving request %v (#%v/%v) failed: %v", self.priority, req, inBatch, inDb, err) + glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err) continue LOOP } batch.Put(entry.key, entry.val) - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] to batch %v '%v' (#%v/%v/%v)", self.priority, req, entry, inBatch, inDb, counter) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter) // if just switched to db mode and not quitting, then launch dbRead // in a parallel go routine to send deliveries from db if inDb == 0 && quit != nil { - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] start dbRead") + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] start dbRead") go self.dbRead(true, counter, deliver) } inDb++ @@ -221,7 +221,7 @@ LOOP: func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch { err := self.db.Write(batch) if err != nil { - glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving batch to db failed: %v", self.priority, err) + glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err) return batch } return new(leveldb.Batch) @@ -295,7 +295,7 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{} continue } del = new(leveldb.Batch) - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v]: new iterator: %x (batch %v, count %v)", self.priority, key, batches, cnt) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt) for n = 0; !useBatches || n < cnt; it.Next() { copy(key, it.Key()) @@ -307,11 +307,11 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{} val := make([]byte, 40) copy(val, it.Value()) entry = &syncDbEntry{key, val} - // glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.priority, self.key.Log(), batches, total, self.dbTotal, self.total) + // glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total) more = fun(entry, self.quit) if !more { // quit received when waiting to deliver entry, the entry will not be deleted - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] batch %v quit after %v/%v items", self.priority, batches, n, cnt) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt) break } // since subsequent batches of the same db session are indexed incrementally diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index e540e236f5..1d94ee4331 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -298,6 +298,7 @@ func (self *syncer) sync() { if state.LastSeenAt < state.SessionAt { state.Last = state.SessionAt glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state) + // blocks until state syncing is finished self.syncState(state) } glog.V(logger.Info).Infof("[BZZ] syncer[%v]: syncing all history complete", self.key.Log()) @@ -316,7 +317,7 @@ func (self *syncer) syncState(state *syncState) { // stop quits both request processor and saves the request cache to disk func (self *syncer) stop() { close(self.quit) - glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: stopand save sync request db backlog", self.key.Log()) + glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: stop and save sync request db backlog", self.key.Log()) for _, db := range self.queues { db.stop() } @@ -358,19 +359,18 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} { IT: for { key := it.Next() - if key != nil { - select { - // blocking until history channel is read from - case history <- storage.Key(key): - n++ - glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n) - state.Latest = key - case <-self.quit: - return - } - } else { + if key == nil { break IT } + select { + // blocking until history channel is read from + case history <- storage.Key(key): + n++ + glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n) + state.Latest = key + case <-self.quit: + return + } } glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n) }() @@ -416,25 +416,31 @@ LOOP: // are checked first - integrity can only be guaranteed if writing // is locked while selecting if priority != High || len(keys) == 0 { + // selection is not needed if the High priority queue has items keys = nil + PRIORITIES: for priority = High; priority >= 0; priority-- { + // the first priority channel that is non-empty will be assigned to keys if len(self.keys[priority]) > 0 { glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading request with priority %v", self.key.Log(), priority) keys = self.keys[priority] - break + break PRIORITIES } + glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low])) + // if the input queue is empty on this level, resort to history if there is any if uint(priority) == histPrior && history != nil { glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key) keys = history - break + break PRIORITIES } } - // if peer ready to receive but nothing to send - if keys == nil && deliveryRequest == nil { - // if no items left and switch to waiting mode - glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: buffers consumed. Waiting", self.key.Log()) - newUnsyncedKeys = self.newUnsyncedKeys - } + } + + // if peer ready to receive but nothing to send + if keys == nil && deliveryRequest == nil { + // if no items left and switch to waiting mode + glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: buffers consumed. Waiting", self.key.Log()) + newUnsyncedKeys = self.newUnsyncedKeys } // send msg iff @@ -447,7 +453,7 @@ LOOP: len(unsynced) > 0 && keys == nil || len(unsynced) == int(self.SyncBatchSize)) { justSynced = false - // listen to requests again + // listen to requests deliveryRequest = self.deliveryRequest newUnsyncedKeys = nil // not care about data until next req comes in // set sync to current counter @@ -458,11 +464,11 @@ LOOP: // send the unsynced keys stateCopy := *state err := self.unsyncedKeys(unsynced, &stateCopy) - self.state = state - glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy) if err != nil { glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: unable to send unsynced keys: %v", err) } + self.state = state + glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy) unsynced = nil keys = nil } @@ -477,7 +483,11 @@ LOOP: // history channel is closed, waiting for new state (called from sync()) syncStates = self.syncStates state.Synced = true // this signals that the current segment is complete - state.synced <- false + select { + case state.synced <- false: + case <-self.quit: + break LOOP + } justSynced = true history = nil } @@ -575,7 +585,7 @@ func (self *syncer) syncDeliveries() { total++ msg, err = self.newStoreRequestMsgData(req) if err != nil { - glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err) + glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err) } else { err = self.store(msg) if err != nil { diff --git a/swarm/services/chequebook/cheque.go b/swarm/services/chequebook/cheque.go index eb2150b468..7bbd2e9392 100644 --- a/swarm/services/chequebook/cheque.go +++ b/swarm/services/chequebook/cheque.go @@ -131,8 +131,10 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva } func (self *Chequebook) setBalanceFromBlockChain() { - balance := self.backend.BalanceAt(self.contractAddr) - self.balance.Set(balance) + balance, err := self.backend.BalanceAt(self.contractAddr) + if err != nil { + self.balance.Set(balance) + } } // LoadChequebook(path, prvKey, backend) loads a chequebook from disk (file path) diff --git a/swarm/services/chequebook/cheque_test.go b/swarm/services/chequebook/cheque_test.go index c74b64be43..44db5091aa 100644 --- a/swarm/services/chequebook/cheque_test.go +++ b/swarm/services/chequebook/cheque_test.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/services/chequebook/contract" ) @@ -42,16 +41,16 @@ func newTestBackend() *testBackend { return &testBackend{SimulatedBackend: backends.NewSimulatedBackend(accs...)} } -func (b *testBackend) GetTxReceipt(txhash common.Hash) *types.Receipt { - return nil +func (b *testBackend) GetTxReceipt(txhash common.Hash) (map[string]interface{}, error) { + return nil, nil } -func (b *testBackend) CodeAt(address common.Address) string { - return "" +func (b *testBackend) CodeAt(address common.Address) (string, error) { + return "", nil } -func (b *testBackend) BalanceAt(address common.Address) *big.Int { - return big.NewInt(0) +func (b *testBackend) BalanceAt(address common.Address) (*big.Int, error) { + return big.NewInt(0), nil } func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) { diff --git a/swarm/services/ens/contract/ens.go b/swarm/services/ens/contract/ens.go index 52611a58f8..b647fa7f56 100644 --- a/swarm/services/ens/contract/ens.go +++ b/swarm/services/ens/contract/ens.go @@ -12,108 +12,108 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// ENSABI is the input ABI used to generate the binding from. -const ENSABI = `[{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"Owners","outputs":[{"name":"","type":"address"}],"type":"function"},{"constant":false,"inputs":[],"name":"kill","outputs":[],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes32"}],"name":"Registry","outputs":[{"name":"","type":"bytes32"}],"type":"function"},{"constant":false,"inputs":[{"name":"host","type":"bytes32"},{"name":"content","type":"bytes32"}],"name":"Set","outputs":[],"type":"function"}]` +// ResolverABI is the input ABI used to generate the binding from. +const ResolverABI = `[{"constant":false,"inputs":[{"name":"rootNodeId","type":"bytes12"},{"name":"name","type":"bytes32[]"}],"name":"deletePrivateRR","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"isPersonalResolver","outputs":[{"name":"","type":"bool"}],"type":"function"},{"constant":false,"inputs":[{"name":"label","type":"bytes32"},{"name":"newOwner","type":"address"}],"name":"setOwner","outputs":[],"type":"function"},{"constant":true,"inputs":[{"name":"id","type":"bytes32"}],"name":"getExtended","outputs":[{"name":"data","type":"bytes"}],"type":"function"},{"constant":false,"inputs":[{"name":"rootNodeId","type":"bytes12"},{"name":"name","type":"string"},{"name":"rtype","type":"bytes16"},{"name":"ttl","type":"uint32"},{"name":"len","type":"uint16"},{"name":"data","type":"bytes32"}],"name":"setRR","outputs":[],"type":"function"},{"constant":false,"inputs":[{"name":"rootNodeId","type":"bytes12"},{"name":"name","type":"bytes32[]"},{"name":"rtype","type":"bytes16"},{"name":"ttl","type":"uint32"},{"name":"len","type":"uint16"},{"name":"data","type":"bytes32"}],"name":"setPrivateRR","outputs":[],"type":"function"},{"constant":true,"inputs":[{"name":"nodeId","type":"bytes12"},{"name":"qtype","type":"bytes32"},{"name":"index","type":"uint16"}],"name":"resolve","outputs":[{"name":"rcode","type":"uint16"},{"name":"rtype","type":"bytes16"},{"name":"ttl","type":"uint32"},{"name":"len","type":"uint16"},{"name":"data","type":"bytes32"}],"type":"function"},{"constant":false,"inputs":[{"name":"label","type":"bytes32"},{"name":"resolver","type":"address"},{"name":"nodeId","type":"bytes12"}],"name":"register","outputs":[],"type":"function"},{"constant":false,"inputs":[{"name":"label","type":"bytes32"},{"name":"resolver","type":"address"},{"name":"nodeId","type":"bytes12"}],"name":"setResolver","outputs":[],"type":"function"},{"constant":false,"inputs":[{"name":"rootNodeId","type":"bytes12"},{"name":"name","type":"string"}],"name":"deleteRR","outputs":[],"type":"function"},{"constant":true,"inputs":[{"name":"label","type":"bytes32"}],"name":"getOwner","outputs":[{"name":"","type":"address"}],"type":"function"},{"constant":true,"inputs":[{"name":"nodeId","type":"bytes12"},{"name":"label","type":"bytes32"}],"name":"findResolver","outputs":[{"name":"rcode","type":"uint16"},{"name":"ttl","type":"uint32"},{"name":"rnode","type":"bytes12"},{"name":"raddress","type":"address"}],"type":"function"}]` -// ENSBin is the compiled bytecode used for deploying new contracts. -const ENSBin = `0x606060405260008054600160a060020a03191633179055610148806100246000396000f3606060405260e060020a60003504633d14b257811461003c57806341c0e1b51461005d578063a0d03d3614610085578063be36e6761461009d575b005b61013c600435600260205260009081526040902054600160a060020a031681565b61003a60005433600160a060020a039081169116141561014657600054600160a060020a0316ff5b61013c60043560016020526000908152604090205481565b61003a600435602435600082815260026020526040812054600160a060020a031614156100e7576040600020805473ffffffffffffffffffffffffffffffffffffffff1916321790555b32600160a060020a03166002600050600084815260200190815260200160002060009054906101000a9004600160a060020a0316600160a060020a0316141561013857600160205260406000208190555b5050565b6060908152602090f35b56` +// ResolverBin is the compiled bytecode used for deploying new contracts. +const ResolverBin = `0x` -// DeployENS deploys a new Ethereum contract, binding an instance of ENS to it. -func DeployENS(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *ENS, error) { - parsed, err := abi.JSON(strings.NewReader(ENSABI)) +// DeployResolver deploys a new Ethereum contract, binding an instance of Resolver to it. +func DeployResolver(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *Resolver, error) { + parsed, err := abi.JSON(strings.NewReader(ResolverABI)) if err != nil { return common.Address{}, nil, nil, err } - address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(ENSBin), backend) + address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(ResolverBin), backend) if err != nil { return common.Address{}, nil, nil, err } - return address, tx, &ENS{ENSCaller: ENSCaller{contract: contract}, ENSTransactor: ENSTransactor{contract: contract}}, nil + return address, tx, &Resolver{ResolverCaller: ResolverCaller{contract: contract}, ResolverTransactor: ResolverTransactor{contract: contract}}, nil } -// ENS is an auto generated Go binding around an Ethereum contract. -type ENS struct { - ENSCaller // Read-only binding to the contract - ENSTransactor // Write-only binding to the contract +// Resolver is an auto generated Go binding around an Ethereum contract. +type Resolver struct { + ResolverCaller // Read-only binding to the contract + ResolverTransactor // Write-only binding to the contract } -// ENSCaller is an auto generated read-only Go binding around an Ethereum contract. -type ENSCaller struct { +// ResolverCaller is an auto generated read-only Go binding around an Ethereum contract. +type ResolverCaller struct { contract *bind.BoundContract // Generic contract wrapper for the low level calls } -// ENSTransactor is an auto generated write-only Go binding around an Ethereum contract. -type ENSTransactor struct { +// ResolverTransactor is an auto generated write-only Go binding around an Ethereum contract. +type ResolverTransactor struct { contract *bind.BoundContract // Generic contract wrapper for the low level calls } -// ENSSession is an auto generated Go binding around an Ethereum contract, +// ResolverSession is an auto generated Go binding around an Ethereum contract, // with pre-set call and transact options. -type ENSSession struct { - Contract *ENS // Generic contract binding to set the session for +type ResolverSession struct { + Contract *Resolver // Generic contract binding to set the session for CallOpts bind.CallOpts // Call options to use throughout this session TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session } -// ENSCallerSession is an auto generated read-only Go binding around an Ethereum contract, +// ResolverCallerSession is an auto generated read-only Go binding around an Ethereum contract, // with pre-set call options. -type ENSCallerSession struct { - Contract *ENSCaller // Generic contract caller binding to set the session for - CallOpts bind.CallOpts // Call options to use throughout this session +type ResolverCallerSession struct { + Contract *ResolverCaller // Generic contract caller binding to set the session for + CallOpts bind.CallOpts // Call options to use throughout this session } -// ENSTransactorSession is an auto generated write-only Go binding around an Ethereum contract, +// ResolverTransactorSession is an auto generated write-only Go binding around an Ethereum contract, // with pre-set transact options. -type ENSTransactorSession struct { - Contract *ENSTransactor // Generic contract transactor binding to set the session for - TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session +type ResolverTransactorSession struct { + Contract *ResolverTransactor // Generic contract transactor binding to set the session for + TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session } -// ENSRaw is an auto generated low-level Go binding around an Ethereum contract. -type ENSRaw struct { - Contract *ENS // Generic contract binding to access the raw methods on +// ResolverRaw is an auto generated low-level Go binding around an Ethereum contract. +type ResolverRaw struct { + Contract *Resolver // Generic contract binding to access the raw methods on } -// ENSCallerRaw is an auto generated low-level read-only Go binding around an Ethereum contract. -type ENSCallerRaw struct { - Contract *ENSCaller // Generic read-only contract binding to access the raw methods on +// ResolverCallerRaw is an auto generated low-level read-only Go binding around an Ethereum contract. +type ResolverCallerRaw struct { + Contract *ResolverCaller // Generic read-only contract binding to access the raw methods on } -// ENSTransactorRaw is an auto generated low-level write-only Go binding around an Ethereum contract. -type ENSTransactorRaw struct { - Contract *ENSTransactor // Generic write-only contract binding to access the raw methods on +// ResolverTransactorRaw is an auto generated low-level write-only Go binding around an Ethereum contract. +type ResolverTransactorRaw struct { + Contract *ResolverTransactor // Generic write-only contract binding to access the raw methods on } -// NewENS creates a new instance of ENS, bound to a specific deployed contract. -func NewENS(address common.Address, backend bind.ContractBackend) (*ENS, error) { - contract, err := bindENS(address, backend.(bind.ContractCaller), backend.(bind.ContractTransactor)) +// NewResolver creates a new instance of Resolver, bound to a specific deployed contract. +func NewResolver(address common.Address, backend bind.ContractBackend) (*Resolver, error) { + contract, err := bindResolver(address, backend.(bind.ContractCaller), backend.(bind.ContractTransactor)) if err != nil { return nil, err } - return &ENS{ENSCaller: ENSCaller{contract: contract}, ENSTransactor: ENSTransactor{contract: contract}}, nil + return &Resolver{ResolverCaller: ResolverCaller{contract: contract}, ResolverTransactor: ResolverTransactor{contract: contract}}, nil } -// NewENSCaller creates a new read-only instance of ENS, bound to a specific deployed contract. -func NewENSCaller(address common.Address, caller bind.ContractCaller) (*ENSCaller, error) { - contract, err := bindENS(address, caller, nil) +// NewResolverCaller creates a new read-only instance of Resolver, bound to a specific deployed contract. +func NewResolverCaller(address common.Address, caller bind.ContractCaller) (*ResolverCaller, error) { + contract, err := bindResolver(address, caller, nil) if err != nil { return nil, err } - return &ENSCaller{contract: contract}, nil + return &ResolverCaller{contract: contract}, nil } -// NewENSTransactor creates a new write-only instance of ENS, bound to a specific deployed contract. -func NewENSTransactor(address common.Address, transactor bind.ContractTransactor) (*ENSTransactor, error) { - contract, err := bindENS(address, nil, transactor) +// NewResolverTransactor creates a new write-only instance of Resolver, bound to a specific deployed contract. +func NewResolverTransactor(address common.Address, transactor bind.ContractTransactor) (*ResolverTransactor, error) { + contract, err := bindResolver(address, nil, transactor) if err != nil { return nil, err } - return &ENSTransactor{contract: contract}, nil + return &ResolverTransactor{contract: contract}, nil } -// bindENS binds a generic wrapper to an already deployed contract. -func bindENS(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) { - parsed, err := abi.JSON(strings.NewReader(ENSABI)) +// bindResolver binds a generic wrapper to an already deployed contract. +func bindResolver(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) { + parsed, err := abi.JSON(strings.NewReader(ResolverABI)) if err != nil { return nil, err } @@ -124,443 +124,353 @@ func bindENS(address common.Address, caller bind.ContractCaller, transactor bind // sets the output to result. The result type might be a single field for simple // returns, a slice of interfaces for anonymous returns and a struct for named // returns. -func (_ENS *ENSRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { - return _ENS.Contract.ENSCaller.contract.Call(opts, result, method, params...) +func (_Resolver *ResolverRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { + return _Resolver.Contract.ResolverCaller.contract.Call(opts, result, method, params...) } // Transfer initiates a plain transaction to move funds to the contract, calling // its default method if one is available. -func (_ENS *ENSRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { - return _ENS.Contract.ENSTransactor.contract.Transfer(opts) +func (_Resolver *ResolverRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { + return _Resolver.Contract.ResolverTransactor.contract.Transfer(opts) } // Transact invokes the (paid) contract method with params as input values. -func (_ENS *ENSRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { - return _ENS.Contract.ENSTransactor.contract.Transact(opts, method, params...) +func (_Resolver *ResolverRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { + return _Resolver.Contract.ResolverTransactor.contract.Transact(opts, method, params...) } // Call invokes the (constant) contract method with params as input values and // sets the output to result. The result type might be a single field for simple // returns, a slice of interfaces for anonymous returns and a struct for named // returns. -func (_ENS *ENSCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { - return _ENS.Contract.contract.Call(opts, result, method, params...) +func (_Resolver *ResolverCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { + return _Resolver.Contract.contract.Call(opts, result, method, params...) } // Transfer initiates a plain transaction to move funds to the contract, calling // its default method if one is available. -func (_ENS *ENSTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { - return _ENS.Contract.contract.Transfer(opts) +func (_Resolver *ResolverTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { + return _Resolver.Contract.contract.Transfer(opts) } // Transact invokes the (paid) contract method with params as input values. -func (_ENS *ENSTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { - return _ENS.Contract.contract.Transact(opts, method, params...) +func (_Resolver *ResolverTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { + return _Resolver.Contract.contract.Transact(opts, method, params...) } -// Owners is a free data retrieval call binding the contract method 0x3d14b257. +// FindResolver is a free data retrieval call binding the contract method 0xedc0277c. // -// Solidity: function Owners( bytes32) constant returns(address) -func (_ENS *ENSCaller) Owners(opts *bind.CallOpts, arg0 [32]byte) (common.Address, error) { +// Solidity: function findResolver(nodeId bytes12, label bytes32) constant returns(rcode uint16, ttl uint32, rnode bytes12, raddress address) +func (_Resolver *ResolverCaller) FindResolver(opts *bind.CallOpts, nodeId [12]byte, label [32]byte) (struct { + Rcode uint16 + Ttl uint32 + Rnode [12]byte + Raddress common.Address +}, error) { + ret := new(struct { + Rcode uint16 + Ttl uint32 + Rnode [12]byte + Raddress common.Address + }) + out := ret + err := _Resolver.contract.Call(opts, out, "findResolver", nodeId, label) + return *ret, err +} + +// FindResolver is a free data retrieval call binding the contract method 0xedc0277c. +// +// Solidity: function findResolver(nodeId bytes12, label bytes32) constant returns(rcode uint16, ttl uint32, rnode bytes12, raddress address) +func (_Resolver *ResolverSession) FindResolver(nodeId [12]byte, label [32]byte) (struct { + Rcode uint16 + Ttl uint32 + Rnode [12]byte + Raddress common.Address +}, error) { + return _Resolver.Contract.FindResolver(&_Resolver.CallOpts, nodeId, label) +} + +// FindResolver is a free data retrieval call binding the contract method 0xedc0277c. +// +// Solidity: function findResolver(nodeId bytes12, label bytes32) constant returns(rcode uint16, ttl uint32, rnode bytes12, raddress address) +func (_Resolver *ResolverCallerSession) FindResolver(nodeId [12]byte, label [32]byte) (struct { + Rcode uint16 + Ttl uint32 + Rnode [12]byte + Raddress common.Address +}, error) { + return _Resolver.Contract.FindResolver(&_Resolver.CallOpts, nodeId, label) +} + +// GetExtended is a free data retrieval call binding the contract method 0x8021061c. +// +// Solidity: function getExtended(id bytes32) constant returns(data bytes) +func (_Resolver *ResolverCaller) GetExtended(opts *bind.CallOpts, id [32]byte) ([]byte, error) { + var ( + ret0 = new([]byte) + ) + out := ret0 + err := _Resolver.contract.Call(opts, out, "getExtended", id) + return *ret0, err +} + +// GetExtended is a free data retrieval call binding the contract method 0x8021061c. +// +// Solidity: function getExtended(id bytes32) constant returns(data bytes) +func (_Resolver *ResolverSession) GetExtended(id [32]byte) ([]byte, error) { + return _Resolver.Contract.GetExtended(&_Resolver.CallOpts, id) +} + +// GetExtended is a free data retrieval call binding the contract method 0x8021061c. +// +// Solidity: function getExtended(id bytes32) constant returns(data bytes) +func (_Resolver *ResolverCallerSession) GetExtended(id [32]byte) ([]byte, error) { + return _Resolver.Contract.GetExtended(&_Resolver.CallOpts, id) +} + +// GetOwner is a free data retrieval call binding the contract method 0xdeb931a2. +// +// Solidity: function getOwner(label bytes32) constant returns(address) +func (_Resolver *ResolverCaller) GetOwner(opts *bind.CallOpts, label [32]byte) (common.Address, error) { var ( ret0 = new(common.Address) ) out := ret0 - err := _ENS.contract.Call(opts, out, "Owners", arg0) + err := _Resolver.contract.Call(opts, out, "getOwner", label) return *ret0, err } -// Owners is a free data retrieval call binding the contract method 0x3d14b257. +// GetOwner is a free data retrieval call binding the contract method 0xdeb931a2. // -// Solidity: function Owners( bytes32) constant returns(address) -func (_ENS *ENSSession) Owners(arg0 [32]byte) (common.Address, error) { - return _ENS.Contract.Owners(&_ENS.CallOpts, arg0) +// Solidity: function getOwner(label bytes32) constant returns(address) +func (_Resolver *ResolverSession) GetOwner(label [32]byte) (common.Address, error) { + return _Resolver.Contract.GetOwner(&_Resolver.CallOpts, label) } -// Owners is a free data retrieval call binding the contract method 0x3d14b257. +// GetOwner is a free data retrieval call binding the contract method 0xdeb931a2. // -// Solidity: function Owners( bytes32) constant returns(address) -func (_ENS *ENSCallerSession) Owners(arg0 [32]byte) (common.Address, error) { - return _ENS.Contract.Owners(&_ENS.CallOpts, arg0) +// Solidity: function getOwner(label bytes32) constant returns(address) +func (_Resolver *ResolverCallerSession) GetOwner(label [32]byte) (common.Address, error) { + return _Resolver.Contract.GetOwner(&_Resolver.CallOpts, label) } -// Registry is a free data retrieval call binding the contract method 0xa0d03d36. +// IsPersonalResolver is a free data retrieval call binding the contract method 0x3f5665e7. // -// Solidity: function Registry( bytes32) constant returns(bytes32) -func (_ENS *ENSCaller) Registry(opts *bind.CallOpts, arg0 [32]byte) ([32]byte, error) { +// Solidity: function isPersonalResolver() constant returns(bool) +func (_Resolver *ResolverCaller) IsPersonalResolver(opts *bind.CallOpts) (bool, error) { var ( - ret0 = new([32]byte) + ret0 = new(bool) ) out := ret0 - err := _ENS.contract.Call(opts, out, "Registry", arg0) + err := _Resolver.contract.Call(opts, out, "isPersonalResolver") return *ret0, err } -// Registry is a free data retrieval call binding the contract method 0xa0d03d36. +// IsPersonalResolver is a free data retrieval call binding the contract method 0x3f5665e7. // -// Solidity: function Registry( bytes32) constant returns(bytes32) -func (_ENS *ENSSession) Registry(arg0 [32]byte) ([32]byte, error) { - return _ENS.Contract.Registry(&_ENS.CallOpts, arg0) +// Solidity: function isPersonalResolver() constant returns(bool) +func (_Resolver *ResolverSession) IsPersonalResolver() (bool, error) { + return _Resolver.Contract.IsPersonalResolver(&_Resolver.CallOpts) } -// Registry is a free data retrieval call binding the contract method 0xa0d03d36. +// IsPersonalResolver is a free data retrieval call binding the contract method 0x3f5665e7. // -// Solidity: function Registry( bytes32) constant returns(bytes32) -func (_ENS *ENSCallerSession) Registry(arg0 [32]byte) ([32]byte, error) { - return _ENS.Contract.Registry(&_ENS.CallOpts, arg0) +// Solidity: function isPersonalResolver() constant returns(bool) +func (_Resolver *ResolverCallerSession) IsPersonalResolver() (bool, error) { + return _Resolver.Contract.IsPersonalResolver(&_Resolver.CallOpts) } -// Set is a paid mutator transaction binding the contract method 0xbe36e676. +// Resolve is a free data retrieval call binding the contract method 0xa16fdafa. // -// Solidity: function Set(host bytes32, content bytes32) returns() -func (_ENS *ENSTransactor) Set(opts *bind.TransactOpts, host [32]byte, content [32]byte) (*types.Transaction, error) { - return _ENS.contract.Transact(opts, "Set", host, content) +// Solidity: function resolve(nodeId bytes12, qtype bytes32, index uint16) constant returns(rcode uint16, rtype bytes16, ttl uint32, len uint16, data bytes32) +func (_Resolver *ResolverCaller) Resolve(opts *bind.CallOpts, nodeId [12]byte, qtype [32]byte, index uint16) (struct { + Rcode uint16 + Rtype [16]byte + Ttl uint32 + Len uint16 + Data [32]byte +}, error) { + ret := new(struct { + Rcode uint16 + Rtype [16]byte + Ttl uint32 + Len uint16 + Data [32]byte + }) + out := ret + err := _Resolver.contract.Call(opts, out, "resolve", nodeId, qtype, index) + return *ret, err } -// Set is a paid mutator transaction binding the contract method 0xbe36e676. +// Resolve is a free data retrieval call binding the contract method 0xa16fdafa. // -// Solidity: function Set(host bytes32, content bytes32) returns() -func (_ENS *ENSSession) Set(host [32]byte, content [32]byte) (*types.Transaction, error) { - return _ENS.Contract.Set(&_ENS.TransactOpts, host, content) +// Solidity: function resolve(nodeId bytes12, qtype bytes32, index uint16) constant returns(rcode uint16, rtype bytes16, ttl uint32, len uint16, data bytes32) +func (_Resolver *ResolverSession) Resolve(nodeId [12]byte, qtype [32]byte, index uint16) (struct { + Rcode uint16 + Rtype [16]byte + Ttl uint32 + Len uint16 + Data [32]byte +}, error) { + return _Resolver.Contract.Resolve(&_Resolver.CallOpts, nodeId, qtype, index) } -// Set is a paid mutator transaction binding the contract method 0xbe36e676. +// Resolve is a free data retrieval call binding the contract method 0xa16fdafa. // -// Solidity: function Set(host bytes32, content bytes32) returns() -func (_ENS *ENSTransactorSession) Set(host [32]byte, content [32]byte) (*types.Transaction, error) { - return _ENS.Contract.Set(&_ENS.TransactOpts, host, content) +// Solidity: function resolve(nodeId bytes12, qtype bytes32, index uint16) constant returns(rcode uint16, rtype bytes16, ttl uint32, len uint16, data bytes32) +func (_Resolver *ResolverCallerSession) Resolve(nodeId [12]byte, qtype [32]byte, index uint16) (struct { + Rcode uint16 + Rtype [16]byte + Ttl uint32 + Len uint16 + Data [32]byte +}, error) { + return _Resolver.Contract.Resolve(&_Resolver.CallOpts, nodeId, qtype, index) } -// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5. +// DeletePrivateRR is a paid mutator transaction binding the contract method 0x1b370194. // -// Solidity: function kill() returns() -func (_ENS *ENSTransactor) Kill(opts *bind.TransactOpts) (*types.Transaction, error) { - return _ENS.contract.Transact(opts, "kill") +// Solidity: function deletePrivateRR(rootNodeId bytes12, name bytes32[]) returns() +func (_Resolver *ResolverTransactor) DeletePrivateRR(opts *bind.TransactOpts, rootNodeId [12]byte, name [][32]byte) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "deletePrivateRR", rootNodeId, name) } -// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5. +// DeletePrivateRR is a paid mutator transaction binding the contract method 0x1b370194. // -// Solidity: function kill() returns() -func (_ENS *ENSSession) Kill() (*types.Transaction, error) { - return _ENS.Contract.Kill(&_ENS.TransactOpts) +// Solidity: function deletePrivateRR(rootNodeId bytes12, name bytes32[]) returns() +func (_Resolver *ResolverSession) DeletePrivateRR(rootNodeId [12]byte, name [][32]byte) (*types.Transaction, error) { + return _Resolver.Contract.DeletePrivateRR(&_Resolver.TransactOpts, rootNodeId, name) } -// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5. +// DeletePrivateRR is a paid mutator transaction binding the contract method 0x1b370194. // -// Solidity: function kill() returns() -func (_ENS *ENSTransactorSession) Kill() (*types.Transaction, error) { - return _ENS.Contract.Kill(&_ENS.TransactOpts) +// Solidity: function deletePrivateRR(rootNodeId bytes12, name bytes32[]) returns() +func (_Resolver *ResolverTransactorSession) DeletePrivateRR(rootNodeId [12]byte, name [][32]byte) (*types.Transaction, error) { + return _Resolver.Contract.DeletePrivateRR(&_Resolver.TransactOpts, rootNodeId, name) } -// MortalABI is the input ABI used to generate the binding from. -const MortalABI = `[{"constant":false,"inputs":[],"name":"kill","outputs":[],"type":"function"}]` - -// MortalBin is the compiled bytecode used for deploying new contracts. -const MortalBin = `0x606060405260008054600160a060020a03191633179055605c8060226000396000f3606060405260e060020a600035046341c0e1b58114601a575b005b60186000543373ffffffffffffffffffffffffffffffffffffffff90811691161415605a5760005473ffffffffffffffffffffffffffffffffffffffff16ff5b56` - -// DeployMortal deploys a new Ethereum contract, binding an instance of Mortal to it. -func DeployMortal(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *Mortal, error) { - parsed, err := abi.JSON(strings.NewReader(MortalABI)) - if err != nil { - return common.Address{}, nil, nil, err - } - address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(MortalBin), backend) - if err != nil { - return common.Address{}, nil, nil, err - } - return address, tx, &Mortal{MortalCaller: MortalCaller{contract: contract}, MortalTransactor: MortalTransactor{contract: contract}}, nil -} - -// Mortal is an auto generated Go binding around an Ethereum contract. -type Mortal struct { - MortalCaller // Read-only binding to the contract - MortalTransactor // Write-only binding to the contract -} - -// MortalCaller is an auto generated read-only Go binding around an Ethereum contract. -type MortalCaller struct { - contract *bind.BoundContract // Generic contract wrapper for the low level calls -} - -// MortalTransactor is an auto generated write-only Go binding around an Ethereum contract. -type MortalTransactor struct { - contract *bind.BoundContract // Generic contract wrapper for the low level calls -} - -// MortalSession is an auto generated Go binding around an Ethereum contract, -// with pre-set call and transact options. -type MortalSession struct { - Contract *Mortal // Generic contract binding to set the session for - CallOpts bind.CallOpts // Call options to use throughout this session - TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session -} - -// MortalCallerSession is an auto generated read-only Go binding around an Ethereum contract, -// with pre-set call options. -type MortalCallerSession struct { - Contract *MortalCaller // Generic contract caller binding to set the session for - CallOpts bind.CallOpts // Call options to use throughout this session -} - -// MortalTransactorSession is an auto generated write-only Go binding around an Ethereum contract, -// with pre-set transact options. -type MortalTransactorSession struct { - Contract *MortalTransactor // Generic contract transactor binding to set the session for - TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session -} - -// MortalRaw is an auto generated low-level Go binding around an Ethereum contract. -type MortalRaw struct { - Contract *Mortal // Generic contract binding to access the raw methods on -} - -// MortalCallerRaw is an auto generated low-level read-only Go binding around an Ethereum contract. -type MortalCallerRaw struct { - Contract *MortalCaller // Generic read-only contract binding to access the raw methods on -} - -// MortalTransactorRaw is an auto generated low-level write-only Go binding around an Ethereum contract. -type MortalTransactorRaw struct { - Contract *MortalTransactor // Generic write-only contract binding to access the raw methods on -} - -// NewMortal creates a new instance of Mortal, bound to a specific deployed contract. -func NewMortal(address common.Address, backend bind.ContractBackend) (*Mortal, error) { - contract, err := bindMortal(address, backend.(bind.ContractCaller), backend.(bind.ContractTransactor)) - if err != nil { - return nil, err - } - return &Mortal{MortalCaller: MortalCaller{contract: contract}, MortalTransactor: MortalTransactor{contract: contract}}, nil -} - -// NewMortalCaller creates a new read-only instance of Mortal, bound to a specific deployed contract. -func NewMortalCaller(address common.Address, caller bind.ContractCaller) (*MortalCaller, error) { - contract, err := bindMortal(address, caller, nil) - if err != nil { - return nil, err - } - return &MortalCaller{contract: contract}, nil -} - -// NewMortalTransactor creates a new write-only instance of Mortal, bound to a specific deployed contract. -func NewMortalTransactor(address common.Address, transactor bind.ContractTransactor) (*MortalTransactor, error) { - contract, err := bindMortal(address, nil, transactor) - if err != nil { - return nil, err - } - return &MortalTransactor{contract: contract}, nil -} - -// bindMortal binds a generic wrapper to an already deployed contract. -func bindMortal(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) { - parsed, err := abi.JSON(strings.NewReader(MortalABI)) - if err != nil { - return nil, err - } - return bind.NewBoundContract(address, parsed, caller, transactor), nil -} - -// Call invokes the (constant) contract method with params as input values and -// sets the output to result. The result type might be a single field for simple -// returns, a slice of interfaces for anonymous returns and a struct for named -// returns. -func (_Mortal *MortalRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { - return _Mortal.Contract.MortalCaller.contract.Call(opts, result, method, params...) -} - -// Transfer initiates a plain transaction to move funds to the contract, calling -// its default method if one is available. -func (_Mortal *MortalRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { - return _Mortal.Contract.MortalTransactor.contract.Transfer(opts) -} - -// Transact invokes the (paid) contract method with params as input values. -func (_Mortal *MortalRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { - return _Mortal.Contract.MortalTransactor.contract.Transact(opts, method, params...) -} - -// Call invokes the (constant) contract method with params as input values and -// sets the output to result. The result type might be a single field for simple -// returns, a slice of interfaces for anonymous returns and a struct for named -// returns. -func (_Mortal *MortalCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { - return _Mortal.Contract.contract.Call(opts, result, method, params...) -} - -// Transfer initiates a plain transaction to move funds to the contract, calling -// its default method if one is available. -func (_Mortal *MortalTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { - return _Mortal.Contract.contract.Transfer(opts) -} - -// Transact invokes the (paid) contract method with params as input values. -func (_Mortal *MortalTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { - return _Mortal.Contract.contract.Transact(opts, method, params...) -} - -// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5. +// DeleteRR is a paid mutator transaction binding the contract method 0xbc06183d. // -// Solidity: function kill() returns() -func (_Mortal *MortalTransactor) Kill(opts *bind.TransactOpts) (*types.Transaction, error) { - return _Mortal.contract.Transact(opts, "kill") +// Solidity: function deleteRR(rootNodeId bytes12, name string) returns() +func (_Resolver *ResolverTransactor) DeleteRR(opts *bind.TransactOpts, rootNodeId [12]byte, name string) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "deleteRR", rootNodeId, name) } -// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5. +// DeleteRR is a paid mutator transaction binding the contract method 0xbc06183d. // -// Solidity: function kill() returns() -func (_Mortal *MortalSession) Kill() (*types.Transaction, error) { - return _Mortal.Contract.Kill(&_Mortal.TransactOpts) +// Solidity: function deleteRR(rootNodeId bytes12, name string) returns() +func (_Resolver *ResolverSession) DeleteRR(rootNodeId [12]byte, name string) (*types.Transaction, error) { + return _Resolver.Contract.DeleteRR(&_Resolver.TransactOpts, rootNodeId, name) } -// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5. +// DeleteRR is a paid mutator transaction binding the contract method 0xbc06183d. // -// Solidity: function kill() returns() -func (_Mortal *MortalTransactorSession) Kill() (*types.Transaction, error) { - return _Mortal.Contract.Kill(&_Mortal.TransactOpts) +// Solidity: function deleteRR(rootNodeId bytes12, name string) returns() +func (_Resolver *ResolverTransactorSession) DeleteRR(rootNodeId [12]byte, name string) (*types.Transaction, error) { + return _Resolver.Contract.DeleteRR(&_Resolver.TransactOpts, rootNodeId, name) } -// OwnedABI is the input ABI used to generate the binding from. -const OwnedABI = `[{"inputs":[],"type":"constructor"}]` - -// OwnedBin is the compiled bytecode used for deploying new contracts. -const OwnedBin = `0x606060405260008054600160a060020a0319163317905560068060226000396000f3606060405200` - -// DeployOwned deploys a new Ethereum contract, binding an instance of Owned to it. -func DeployOwned(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *Owned, error) { - parsed, err := abi.JSON(strings.NewReader(OwnedABI)) - if err != nil { - return common.Address{}, nil, nil, err - } - address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(OwnedBin), backend) - if err != nil { - return common.Address{}, nil, nil, err - } - return address, tx, &Owned{OwnedCaller: OwnedCaller{contract: contract}, OwnedTransactor: OwnedTransactor{contract: contract}}, nil +// Register is a paid mutator transaction binding the contract method 0xa1f8f8f0. +// +// Solidity: function register(label bytes32, resolver address, nodeId bytes12) returns() +func (_Resolver *ResolverTransactor) Register(opts *bind.TransactOpts, label [32]byte, resolver common.Address, nodeId [12]byte) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "register", label, resolver, nodeId) } -// Owned is an auto generated Go binding around an Ethereum contract. -type Owned struct { - OwnedCaller // Read-only binding to the contract - OwnedTransactor // Write-only binding to the contract +// Register is a paid mutator transaction binding the contract method 0xa1f8f8f0. +// +// Solidity: function register(label bytes32, resolver address, nodeId bytes12) returns() +func (_Resolver *ResolverSession) Register(label [32]byte, resolver common.Address, nodeId [12]byte) (*types.Transaction, error) { + return _Resolver.Contract.Register(&_Resolver.TransactOpts, label, resolver, nodeId) } -// OwnedCaller is an auto generated read-only Go binding around an Ethereum contract. -type OwnedCaller struct { - contract *bind.BoundContract // Generic contract wrapper for the low level calls +// Register is a paid mutator transaction binding the contract method 0xa1f8f8f0. +// +// Solidity: function register(label bytes32, resolver address, nodeId bytes12) returns() +func (_Resolver *ResolverTransactorSession) Register(label [32]byte, resolver common.Address, nodeId [12]byte) (*types.Transaction, error) { + return _Resolver.Contract.Register(&_Resolver.TransactOpts, label, resolver, nodeId) } -// OwnedTransactor is an auto generated write-only Go binding around an Ethereum contract. -type OwnedTransactor struct { - contract *bind.BoundContract // Generic contract wrapper for the low level calls +// SetOwner is a paid mutator transaction binding the contract method 0x5b0fc9c3. +// +// Solidity: function setOwner(label bytes32, newOwner address) returns() +func (_Resolver *ResolverTransactor) SetOwner(opts *bind.TransactOpts, label [32]byte, newOwner common.Address) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "setOwner", label, newOwner) } -// OwnedSession is an auto generated Go binding around an Ethereum contract, -// with pre-set call and transact options. -type OwnedSession struct { - Contract *Owned // Generic contract binding to set the session for - CallOpts bind.CallOpts // Call options to use throughout this session - TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session +// SetOwner is a paid mutator transaction binding the contract method 0x5b0fc9c3. +// +// Solidity: function setOwner(label bytes32, newOwner address) returns() +func (_Resolver *ResolverSession) SetOwner(label [32]byte, newOwner common.Address) (*types.Transaction, error) { + return _Resolver.Contract.SetOwner(&_Resolver.TransactOpts, label, newOwner) } -// OwnedCallerSession is an auto generated read-only Go binding around an Ethereum contract, -// with pre-set call options. -type OwnedCallerSession struct { - Contract *OwnedCaller // Generic contract caller binding to set the session for - CallOpts bind.CallOpts // Call options to use throughout this session +// SetOwner is a paid mutator transaction binding the contract method 0x5b0fc9c3. +// +// Solidity: function setOwner(label bytes32, newOwner address) returns() +func (_Resolver *ResolverTransactorSession) SetOwner(label [32]byte, newOwner common.Address) (*types.Transaction, error) { + return _Resolver.Contract.SetOwner(&_Resolver.TransactOpts, label, newOwner) } -// OwnedTransactorSession is an auto generated write-only Go binding around an Ethereum contract, -// with pre-set transact options. -type OwnedTransactorSession struct { - Contract *OwnedTransactor // Generic contract transactor binding to set the session for - TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session +// SetPrivateRR is a paid mutator transaction binding the contract method 0x91c8e7b9. +// +// Solidity: function setPrivateRR(rootNodeId bytes12, name bytes32[], rtype bytes16, ttl uint32, len uint16, data bytes32) returns() +func (_Resolver *ResolverTransactor) SetPrivateRR(opts *bind.TransactOpts, rootNodeId [12]byte, name [][32]byte, rtype [16]byte, ttl uint32, len uint16, data [32]byte) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "setPrivateRR", rootNodeId, name, rtype, ttl, len, data) } -// OwnedRaw is an auto generated low-level Go binding around an Ethereum contract. -type OwnedRaw struct { - Contract *Owned // Generic contract binding to access the raw methods on +// SetPrivateRR is a paid mutator transaction binding the contract method 0x91c8e7b9. +// +// Solidity: function setPrivateRR(rootNodeId bytes12, name bytes32[], rtype bytes16, ttl uint32, len uint16, data bytes32) returns() +func (_Resolver *ResolverSession) SetPrivateRR(rootNodeId [12]byte, name [][32]byte, rtype [16]byte, ttl uint32, len uint16, data [32]byte) (*types.Transaction, error) { + return _Resolver.Contract.SetPrivateRR(&_Resolver.TransactOpts, rootNodeId, name, rtype, ttl, len, data) } -// OwnedCallerRaw is an auto generated low-level read-only Go binding around an Ethereum contract. -type OwnedCallerRaw struct { - Contract *OwnedCaller // Generic read-only contract binding to access the raw methods on +// SetPrivateRR is a paid mutator transaction binding the contract method 0x91c8e7b9. +// +// Solidity: function setPrivateRR(rootNodeId bytes12, name bytes32[], rtype bytes16, ttl uint32, len uint16, data bytes32) returns() +func (_Resolver *ResolverTransactorSession) SetPrivateRR(rootNodeId [12]byte, name [][32]byte, rtype [16]byte, ttl uint32, len uint16, data [32]byte) (*types.Transaction, error) { + return _Resolver.Contract.SetPrivateRR(&_Resolver.TransactOpts, rootNodeId, name, rtype, ttl, len, data) } -// OwnedTransactorRaw is an auto generated low-level write-only Go binding around an Ethereum contract. -type OwnedTransactorRaw struct { - Contract *OwnedTransactor // Generic write-only contract binding to access the raw methods on +// SetRR is a paid mutator transaction binding the contract method 0x8bba944d. +// +// Solidity: function setRR(rootNodeId bytes12, name string, rtype bytes16, ttl uint32, len uint16, data bytes32) returns() +func (_Resolver *ResolverTransactor) SetRR(opts *bind.TransactOpts, rootNodeId [12]byte, name string, rtype [16]byte, ttl uint32, len uint16, data [32]byte) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "setRR", rootNodeId, name, rtype, ttl, len, data) } -// NewOwned creates a new instance of Owned, bound to a specific deployed contract. -func NewOwned(address common.Address, backend bind.ContractBackend) (*Owned, error) { - contract, err := bindOwned(address, backend.(bind.ContractCaller), backend.(bind.ContractTransactor)) - if err != nil { - return nil, err - } - return &Owned{OwnedCaller: OwnedCaller{contract: contract}, OwnedTransactor: OwnedTransactor{contract: contract}}, nil +// SetRR is a paid mutator transaction binding the contract method 0x8bba944d. +// +// Solidity: function setRR(rootNodeId bytes12, name string, rtype bytes16, ttl uint32, len uint16, data bytes32) returns() +func (_Resolver *ResolverSession) SetRR(rootNodeId [12]byte, name string, rtype [16]byte, ttl uint32, len uint16, data [32]byte) (*types.Transaction, error) { + return _Resolver.Contract.SetRR(&_Resolver.TransactOpts, rootNodeId, name, rtype, ttl, len, data) } -// NewOwnedCaller creates a new read-only instance of Owned, bound to a specific deployed contract. -func NewOwnedCaller(address common.Address, caller bind.ContractCaller) (*OwnedCaller, error) { - contract, err := bindOwned(address, caller, nil) - if err != nil { - return nil, err - } - return &OwnedCaller{contract: contract}, nil +// SetRR is a paid mutator transaction binding the contract method 0x8bba944d. +// +// Solidity: function setRR(rootNodeId bytes12, name string, rtype bytes16, ttl uint32, len uint16, data bytes32) returns() +func (_Resolver *ResolverTransactorSession) SetRR(rootNodeId [12]byte, name string, rtype [16]byte, ttl uint32, len uint16, data [32]byte) (*types.Transaction, error) { + return _Resolver.Contract.SetRR(&_Resolver.TransactOpts, rootNodeId, name, rtype, ttl, len, data) } -// NewOwnedTransactor creates a new write-only instance of Owned, bound to a specific deployed contract. -func NewOwnedTransactor(address common.Address, transactor bind.ContractTransactor) (*OwnedTransactor, error) { - contract, err := bindOwned(address, nil, transactor) - if err != nil { - return nil, err - } - return &OwnedTransactor{contract: contract}, nil +// SetResolver is a paid mutator transaction binding the contract method 0xa9f2a1b2. +// +// Solidity: function setResolver(label bytes32, resolver address, nodeId bytes12) returns() +func (_Resolver *ResolverTransactor) SetResolver(opts *bind.TransactOpts, label [32]byte, resolver common.Address, nodeId [12]byte) (*types.Transaction, error) { + return _Resolver.contract.Transact(opts, "setResolver", label, resolver, nodeId) } -// bindOwned binds a generic wrapper to an already deployed contract. -func bindOwned(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) { - parsed, err := abi.JSON(strings.NewReader(OwnedABI)) - if err != nil { - return nil, err - } - return bind.NewBoundContract(address, parsed, caller, transactor), nil +// SetResolver is a paid mutator transaction binding the contract method 0xa9f2a1b2. +// +// Solidity: function setResolver(label bytes32, resolver address, nodeId bytes12) returns() +func (_Resolver *ResolverSession) SetResolver(label [32]byte, resolver common.Address, nodeId [12]byte) (*types.Transaction, error) { + return _Resolver.Contract.SetResolver(&_Resolver.TransactOpts, label, resolver, nodeId) } -// Call invokes the (constant) contract method with params as input values and -// sets the output to result. The result type might be a single field for simple -// returns, a slice of interfaces for anonymous returns and a struct for named -// returns. -func (_Owned *OwnedRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { - return _Owned.Contract.OwnedCaller.contract.Call(opts, result, method, params...) -} - -// Transfer initiates a plain transaction to move funds to the contract, calling -// its default method if one is available. -func (_Owned *OwnedRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { - return _Owned.Contract.OwnedTransactor.contract.Transfer(opts) -} - -// Transact invokes the (paid) contract method with params as input values. -func (_Owned *OwnedRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { - return _Owned.Contract.OwnedTransactor.contract.Transact(opts, method, params...) -} - -// Call invokes the (constant) contract method with params as input values and -// sets the output to result. The result type might be a single field for simple -// returns, a slice of interfaces for anonymous returns and a struct for named -// returns. -func (_Owned *OwnedCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { - return _Owned.Contract.contract.Call(opts, result, method, params...) -} - -// Transfer initiates a plain transaction to move funds to the contract, calling -// its default method if one is available. -func (_Owned *OwnedTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { - return _Owned.Contract.contract.Transfer(opts) -} - -// Transact invokes the (paid) contract method with params as input values. -func (_Owned *OwnedTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { - return _Owned.Contract.contract.Transact(opts, method, params...) +// SetResolver is a paid mutator transaction binding the contract method 0xa9f2a1b2. +// +// Solidity: function setResolver(label bytes32, resolver address, nodeId bytes12) returns() +func (_Resolver *ResolverTransactorSession) SetResolver(label [32]byte, resolver common.Address, nodeId [12]byte) (*types.Transaction, error) { + return _Resolver.Contract.SetResolver(&_Resolver.TransactOpts, label, resolver, nodeId) } diff --git a/swarm/services/ens/contract/ens.sol b/swarm/services/ens/contract/ens.sol index 9ef19c5e80..7e0f3e73bb 100644 --- a/swarm/services/ens/contract/ens.sol +++ b/swarm/services/ens/contract/ens.sol @@ -1,19 +1,31 @@ -import "mortal"; -/// @title Swarm Distributed Preimage Archive -/// @author Viktor Tron -contract ENS is mortal -{ +/** + * ENS resolver interface. + */ +contract Resolver { + bytes32 constant TYPE_STAR = "*"; + + // Response codes. + uint16 constant RCODE_OK = 0; + uint16 constant RCODE_NXDOMAIN = 3; - mapping (bytes32 => bytes32) public Registry; - mapping (bytes32 => address) public Owners; + // These methods are shared by all resolvers + function findResolver(bytes12 nodeId, bytes32 label) constant + returns (uint16 rcode, uint32 ttl, bytes12 rnode, address raddress); + function resolve(bytes12 nodeId, bytes32 qtype, uint16 index) constant + returns (uint16 rcode, bytes16 rtype, uint32 ttl, uint16 len, + bytes32 data); + function getExtended(bytes32 id) constant returns (bytes data); - function Set(bytes32 host, bytes32 content) { - if (Owners[host] == 0x0) { - Owners[host] = tx.origin; - } - if (Owners[host] == tx.origin) { - Registry[host] = content; - } - } + // These methods are implemented by personal resolvers + function isPersonalResolver() constant returns (bool); + function setRR(bytes12 rootNodeId, string name, bytes16 rtype, uint32 ttl, uint16 len, bytes32 data); + function setPrivateRR(bytes12 rootNodeId, bytes32[] name, bytes16 rtype, uint32 ttl, uint16 len, bytes32 data); + function deleteRR(bytes12 rootNodeId, string name); + function deletePrivateRR(bytes12 rootNodeId, bytes32[] name); -} \ No newline at end of file + // These methods are implemented by open registrar implementations. + function register(bytes32 label, address resolver, bytes12 nodeId); + function setOwner(bytes32 label, address newOwner); + function setResolver(bytes32 label, address resolver, bytes12 nodeId); + function getOwner(bytes32 label) constant returns (address); +} diff --git a/swarm/services/ens/ens.go b/swarm/services/ens/ens.go index 715d06d0ea..65022982c6 100644 --- a/swarm/services/ens/ens.go +++ b/swarm/services/ens/ens.go @@ -3,82 +3,183 @@ package ens import ( "fmt" - "math/big" "regexp" + "strings" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/services/ens/contract" "github.com/ethereum/go-ethereum/swarm/storage" ) var domainAndVersion = regexp.MustCompile("[@:;,]+") +var qtypeChash = [32]byte{ 0x43, 0x48, 0x41, 0x53, 0x48} +var rtypeChash = [16]byte{ 0x43, 0x48, 0x41, 0x53, 0x48} // swarm domain name registry and resolver // the ENS instance can be directly wrapped in rpc.Api type ENS struct { - *contract.ENSSession + transactOpts *bind.TransactOpts; + contractBackend bind.ContractBackend; + rootAddress common.Address; } -// NewENS creates a proxy instance wrapping the abigen interface to the ENS contract -// using the transaction options passed as first argument, it sets up a session func NewENS(transactOpts *bind.TransactOpts, contractAddr common.Address, contractBackend bind.ContractBackend) *ENS { - ens, err := contract.NewENS(contractAddr, contractBackend) - if err != nil { - glog.V(logger.Debug).Infof("error setting up name server on %v, skipping: %v", contractAddr.Hex(), err) - } return &ENS{ - &contract.ENSSession{ - Contract: ens, - TransactOpts: *transactOpts, - }, + transactOpts: transactOpts, + contractBackend: contractBackend, + rootAddress: contractAddr, } } -// Register(name, hash ) -//involves sending a transaction (sent by sender specified as From of Transact) -func (self *ENS) Register(name string, hash common.Hash) (*types.Transaction, error) { - namehash := crypto.Sha3Hash([]byte(name)) - owner, err := self.Owners(namehash) +func (self *ENS) newResolver(contractAddr common.Address) (*contract.ResolverSession, error) { + resolver, err := contract.NewResolver(contractAddr, self.contractBackend) if err != nil { - return nil, fmt.Errorf("error registering '%s': %v", name, err) + return nil, err } - if (owner != common.Address{} && owner != self.TransactOpts.From) { - return nil, fmt.Errorf("error registering '%s': already set as %", name) - } - glog.V(logger.Debug).Infof("[ENS]: host '%s' (hash: '%v') to be registered as '%v'", name, namehash.Hex(), hash.Hex()) - return self.Set(namehash, hash) -} - -func (self *ENS) WhoseIs(name string) (common.Address, error) { - namehash := crypto.Sha3Hash([]byte(name)) - return self.Owners(namehash) + return &contract.ResolverSession{ + Contract: resolver, + TransactOpts: *self.transactOpts, + }, nil } // resolve is a non-tranasctional call, returns hash as storage.Key func (self *ENS) Resolve(hostPort string) (storage.Key, error) { host := hostPort - var version *big.Int parts := domainAndVersion.Split(host, 3) if len(parts) > 1 && parts[1] != "" { host = parts[0] - version = common.Big(parts[1]) } - hash := crypto.Sha3Hash([]byte(host)) - _ = version - // hash, err = self.registrar.Resolver(version).HashToHash(hostHash) - hash, err := self.Registry(hash) - if err != nil { - return nil, fmt.Errorf("error resolving '%v': %v", hash.Hex(), err) - } - if (hash == common.Hash{}) { - return nil, fmt.Errorf("unable to resolve '%v': not found", hash) - } - contentHash := storage.Key(hash.Bytes()) - glog.V(logger.Debug).Infof("[ENS] resolve host '%v' to contentHash: '%v'", hash, contentHash) - return contentHash, nil + return self.resolveName(self.rootAddress, host) +} + +func (self *ENS) nextResolver(resolver *contract.ResolverSession, nodeId [12]byte, label string) (*contract.ResolverSession, [12]byte, error) { + hash := crypto.Sha3Hash([]byte(label)) + ret, err := resolver.FindResolver(nodeId, hash) + if err != nil { + err = fmt.Errorf("error resolving label '%v': %v", label, err) + return nil, [12]byte{}, err + } + if ret.Rcode != 0 { + err = fmt.Errorf("error resolving label '%v': got response code %v", label, ret.Rcode) + return nil, [12]byte{}, err + } + nodeId = ret.Rnode; + resolver, err = self.newResolver(ret.Raddress) + if err != nil { + return nil, [12]byte{}, err + } + + return resolver, nodeId, nil +} + +func (self *ENS) findResolver(rootAddress common.Address, host string) (*contract.ResolverSession, [12]byte, error) { + resolver, err := self.newResolver(self.rootAddress) + if err != nil { + return nil, [12]byte{}, err + } + + if len(host) == 0 { + return resolver, [12]byte{}, nil + } + + labels := strings.Split(host, ".") + + var nodeId [12]byte + for i := len(labels) - 1; i >= 0; i-- { + var err error + resolver, nodeId, err = self.nextResolver(resolver, nodeId, labels[i]) + if err != nil { + return nil, [12]byte{}, err + } + } + + return resolver, nodeId, nil +} + +func (self *ENS) resolveName(rootAddress common.Address, host string) (storage.Key, error) { + resolver, nodeId, err := self.findResolver(rootAddress, host) + if err != nil { + return nil, err + } + + ret, err := resolver.Resolve(nodeId, qtypeChash, 0) + if err != nil { + return nil, fmt.Errorf("error looking up RR on '%v': %v", host, err) + } + if ret.Rcode != 0 { + return nil, fmt.Errorf("error looking up RR on '%v': got response code %v", host, ret.Rcode) + } + return storage.Key(ret.Data[:]), nil +} + +/** + * Registers a new domain name for the caller, making them the owner of the new name. + */ +func (self *ENS) Register(name string, resolverAddress common.Address) (*types.Transaction, error) { + // Find the resolver that we should register with (the one that controls the parent domain) + parts := strings.SplitN(name, ".", 2) + + baseName := "" + if len(parts) > 1 { + baseName = parts[1] + } + + resolver, nodeId, err := self.findResolver(self.rootAddress, baseName) + if err != nil { + return nil, err + } + if nodeId != [12]byte{} { + return nil, fmt.Errorf("cannot register domains on %v: not a root node", baseName) + } + + // Send it a register transaction + hash := crypto.Sha3Hash([]byte(parts[0])) + return resolver.Register(hash, resolverAddress, [12]byte{}) +} + +/** + * Steps through name components until it finds a PersonalResolver contract. + * Returns the resolver, the node ID, and the remaining name components. + */ +func (self *ENS) findPersonalResolver(name string) (*contract.ResolverSession, [12]byte, string, error) { + var nodeId [12]byte + + resolver, err := self.newResolver(self.rootAddress) + if err != nil { + return nil, [12]byte{}, "", err + } + + labels := strings.Split(name, ".") + + for i := len(labels) - 1; i >= 0; i-- { + if personal, _ := resolver.IsPersonalResolver(); personal { + return resolver, nodeId, strings.Join(labels[0:i + 1], "."), nil + } + + resolver, nodeId, err = self.nextResolver(resolver, nodeId, labels[i]) + if err != nil { + return nil, [12]byte{}, "", err + } + } + + if personal, _ := resolver.IsPersonalResolver(); !personal { + return nil, [12]byte{}, "", fmt.Errorf("Personal resolver not found in any name component") + } else { + return resolver, nodeId, "", nil + } +} + +/** + * Sets the content hash associated with a name. + */ +func (self *ENS) SetContentHash(name string, hash common.Hash) (*types.Transaction, error) { + resolver, nodeId, name, err := self.findPersonalResolver(name) + if err != nil { + return nil, err + } + + return resolver.SetRR(nodeId, name, rtypeChash, 3600, 20, [32]byte(hash)) } diff --git a/swarm/services/ens/ens_test.go b/swarm/services/ens/ens_test.go index 0b99730b46..7480d802d1 100644 --- a/swarm/services/ens/ens_test.go +++ b/swarm/services/ens/ens_test.go @@ -10,9 +10,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/services/ens/contract" ) +func init() { + glog.SetV(6) + glog.SetToStderr(true) +} + var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") name = "my name on ENS" @@ -23,7 +29,7 @@ var ( func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) { deployTransactor := bind.NewKeyedTransactor(prvKey) deployTransactor.Value = amount - addr, _, _, err := contract.DeployENS(deployTransactor, backend) + addr, _, _, err := contract.DeployResolver(deployTransactor, backend) if err != nil { return common.Address{}, err } @@ -38,12 +44,25 @@ func TestENS(t *testing.T) { if err != nil { t.Fatalf("expected no error, got %v", err) } + + resolverAddr, err := deploy(key, big.NewInt(0), contractBackend) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + ens := NewENS(transactOpts, contractAddr, contractBackend) - _, err = ens.Register(name, hash) + _, err = ens.Register(name, resolverAddr) if err != nil { t.Fatalf("expected no error, got %v", err) } contractBackend.Commit() + + _, err = ens.SetContentHash(name, hash) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + contractBackend.Commit() + vhost, err := ens.Resolve(name) if err != nil { t.Fatalf("expected no error, got %v", err) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index c19bb622ec..490f5b432e 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -2,10 +2,11 @@ package storage import ( "encoding/binary" + "errors" "fmt" + "hash" "io" "sync" - "time" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -37,11 +38,9 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} */ const ( - // defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash - defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash + defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash + // defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash defaultBranches int64 = 128 - joinTimeout = 120 // second - splitTimeout = 120 // second // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes // chunksize int64 = branches * hashSize // chunk is defined as this ) @@ -55,130 +54,107 @@ The hashing itself does use extra copies and allocation though, since it does ne */ type ChunkerParams struct { - Branches int64 - Hash string - JoinTimeout time.Duration - SplitTimeout time.Duration + Branches int64 + Hash string } func NewChunkerParams() *ChunkerParams { return &ChunkerParams{ - Branches: defaultBranches, - Hash: defaultHash, - JoinTimeout: joinTimeout, - SplitTimeout: splitTimeout, + Branches: defaultBranches, + Hash: defaultHash, } } type TreeChunker struct { - branches int64 - hashFunc Hasher - joinTimeout time.Duration - splitTimeout time.Duration + branches int64 + hashFunc Hasher // calculated - hashSize int64 // self.hashFunc.New().Size() - chunkSize int64 // hashSize* branches + hashSize int64 // self.hashFunc.New().Size() + chunkSize int64 // hashSize* branches + workerCount int } func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) { self = &TreeChunker{} self.hashFunc = MakeHashFunc(params.Hash) self.branches = params.Branches - self.joinTimeout = params.JoinTimeout * time.Second - self.splitTimeout = params.SplitTimeout * time.Second self.hashSize = int64(self.hashFunc().Size()) self.chunkSize = self.hashSize * self.branches + self.workerCount = 1 return } -func (self *TreeChunker) KeySize() int64 { - return self.hashSize -} +// func (self *TreeChunker) KeySize() int64 { +// return self.hashSize +// } // String() for pretty printing func (self *Chunk) String() string { return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData)) } -// The treeChunkers own Hash hashes together -// - the size (of the subtree encoded in the Chunk) -// - the Chunk, ie. the contents read from the input reader -func (self *TreeChunker) Hash(input []byte) []byte { - hasher := self.hashFunc() - hasher.Write(input) - return hasher.Sum(nil) +type hashJob struct { + key Key + chunk []byte + size int64 + parentWg *sync.WaitGroup } -func (self *TreeChunker) Split(key Key, data SectionReader, chunkC chan *Chunk, swg *sync.WaitGroup) (errC chan error) { - - if swg != nil { - swg.Add(1) - defer swg.Done() - } +func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { if self.chunkSize <= 0 { panic("chunker must be initialised") } - if int64(len(key)) != self.hashSize { - panic(fmt.Sprintf("root key buffer must be allocated byte slice of length %d", self.hashSize)) + jobC := make(chan *hashJob, 2*processors) + wg := &sync.WaitGroup{} + errC := make(chan error) + + // wwg = workers waitgroup keeps track of hashworkers spawned by this split call + if wwg != nil { + wwg.Add(1) + } + go self.hashWorker(jobC, chunkC, errC, swg, wwg) + + depth := 0 + treeSize := self.chunkSize + + // takes lowest depth such that chunksize*HashCount^(depth+1) > size + // power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree. + for ; treeSize < size; treeSize *= self.branches { + depth++ } - wg := &sync.WaitGroup{} - errC = make(chan error) - rerrC := make(chan error) - timeout := time.After(self.splitTimeout) - + key := make([]byte, self.hashFunc().Size()) + // glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth) + // this waitgroup member is released after the root hash is calculated wg.Add(1) - go func() { - - depth := 0 - treeSize := self.chunkSize - size := data.Size() - // takes lowest depth such that chunksize*HashCount^(depth+1) > size - // power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree. - - for ; treeSize < size; treeSize *= self.branches { - depth++ - } - - // glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth) - - //launch actual recursive function passing the workgroup - self.split(depth, treeSize/self.branches, key, data, chunkC, rerrC, wg, swg) - }() + //launch actual recursive function passing the waitgroups + go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, wg, swg, wwg) // closes internal error channel if all subprocesses in the workgroup finished go func() { + // waiting for all threads to finish wg.Wait() - close(rerrC) - - }() - - // waiting for request to end with wg finishing, error, or timeout - go func() { - select { - case err := <-rerrC: - if err != nil { - errC <- err - } // otherwise splitting is complete - case <-timeout: - errC <- fmt.Errorf("split time out") + // if storage waitgroup is non-nil, we wait for storage to finish too + if swg != nil { + // glog.V(logger.Detail).Infof("Waiting for storage to finish") + swg.Wait() } close(errC) }() - return + select { + case err := <-errC: + if err != nil { + return nil, err + } + // + } + return key, nil } -func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionReader, chunkC chan *Chunk, errc chan error, parentWg *sync.WaitGroup, swg *sync.WaitGroup) { - - defer parentWg.Done() - - size := data.Size() - var newChunk *Chunk - var hash Key - // glog.V(logger.Detail).Infof("[BZZ] depth: %v, max subtree size: %v, data size: %v", depth, treeSize, size) +func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, parentWg, swg, wwg *sync.WaitGroup) { for depth > 0 && size < treeSize { treeSize /= self.branches @@ -187,185 +163,233 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionR if depth == 0 { // leaf nodes -> content chunks - chunkData := make([]byte, data.Size()+8) + chunkData := make([]byte, size+8) binary.LittleEndian.PutUint64(chunkData[0:8], uint64(size)) - data.ReadAt(chunkData[8:], 0) - hash = self.Hash(chunkData) - // glog.V(logger.Detail).Infof("[BZZ] content chunk: max subtree size: %v, data size: %v", treeSize, size) - newChunk = &Chunk{ - Key: hash, - SData: chunkData, - Size: size, + data.Read(chunkData[8:]) + select { + case jobC <- &hashJob{key, chunkData, size, parentWg}: + case <-errC: } - } else { - // intermediate chunk containing child nodes hashes - branchCnt := int64((size + treeSize - 1) / treeSize) - // glog.V(logger.Detail).Infof("[BZZ] intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size) + // glog.V(logger.Detail).Infof("[BZZ] read %v", size) + return + } + // intermediate chunk containing child nodes hashes + branchCnt := int64((size + treeSize - 1) / treeSize) + // glog.V(logger.Detail).Infof("[BZZ] intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size) - var chunk []byte = make([]byte, branchCnt*self.hashSize+8) - var pos, i int64 + var chunk []byte = make([]byte, branchCnt*self.hashSize+8) + var pos, i int64 - binary.LittleEndian.PutUint64(chunk[0:8], uint64(size)) + binary.LittleEndian.PutUint64(chunk[0:8], uint64(size)) - childrenWg := &sync.WaitGroup{} - var secSize int64 - for i < branchCnt { - // the last item can have shorter data - if size-pos < treeSize { - secSize = size - pos - } else { - secSize = treeSize + childrenWg := &sync.WaitGroup{} + var secSize int64 + for i < branchCnt { + // the last item can have shorter data + if size-pos < treeSize { + secSize = size - pos + } else { + secSize = treeSize + } + // the hash of that data + subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize] + + childrenWg.Add(1) + self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, childrenWg, swg, wwg) + + i++ + pos += treeSize + } + // wait for all the children to complete calculating their hashes and copying them onto sections of the chunk + // parentWg.Add(1) + // go func() { + childrenWg.Wait() + if len(jobC) > self.workerCount && self.workerCount < processors { + if wwg != nil { + wwg.Add(1) + } + self.workerCount++ + go self.hashWorker(jobC, chunkC, errC, swg, wwg) + } + select { + case jobC <- &hashJob{key, chunk, size, parentWg}: + case <-errC: + } +} + +func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, swg, wwg *sync.WaitGroup) { + hasher := self.hashFunc() + if wwg != nil { + defer wwg.Done() + } + for { + select { + + case job, ok := <-jobC: + if !ok { + return } - // take the section of the data encoded in the subTree - subTreeData := NewChunkReader(data, pos, secSize) - // the hash of that data - subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize] - - childrenWg.Add(1) - go self.split(depth-1, treeSize/self.branches, subTreeKey, subTreeData, chunkC, errc, childrenWg, swg) - - i++ - pos += treeSize - } - // wait for all the children to complete calculating their hashes and copying them onto sections of the chunk - childrenWg.Wait() - // now we got the hashes in the chunk, then hash the chunks - hash = self.Hash(chunk) - newChunk = &Chunk{ - Key: hash, - SData: chunk, - Size: size, - wg: swg, + // now we got the hashes in the chunk, then hash the chunks + hasher.Reset() + self.hashChunk(hasher, job, chunkC, swg) + // glog.V(logger.Detail).Infof("[BZZ] hash chunk (%v)", job.size) + case <-errC: + return } + } +} +// The treeChunkers own Hash hashes together +// - the size (of the subtree encoded in the Chunk) +// - the Chunk, ie. the contents read from the input reader +func (self *TreeChunker) hashChunk(hasher hash.Hash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) { + hasher.Write(job.chunk) + h := hasher.Sum(nil) + newChunk := &Chunk{ + Key: h, + SData: job.chunk, + Size: job.size, + wg: swg, + } + + // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) + copy(job.key, h) + // send off new chunk to storage + if chunkC != nil { if swg != nil { swg.Add(1) } } - - // send off new chunk to storage + job.parentWg.Done() if chunkC != nil { chunkC <- newChunk } - // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)x - copy(key, hash) - -} - -func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) SectionReader { - - return &LazyChunkReader{ - key: key, - chunkC: chunkC, - quitC: make(chan bool), - errC: make(chan error), - chunker: self, - } } // LazyChunkReader implements LazySectionReader type LazyChunkReader struct { - key Key // root key - chunkC chan *Chunk // chunk channel to send retrieve requests on - size int64 // size of the entire subtree - off int64 // offset - quitC chan bool // channel to abort retrieval - errC chan error // error channel to monitor retrieve errors - chunker *TreeChunker // needs TreeChunker params TODO: should just take - // the chunkSize, branches etc as params + key Key // root key + chunkC chan *Chunk // chunk channel to send retrieve requests on + chunk *Chunk // size of the entire subtree + off int64 // offset + chunkSize int64 // inherit from chunker + branches int64 // inherit from chunker + hashSize int64 // inherit from chunker } -func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { - self.errC = make(chan error) - chunk := &Chunk{ - Key: self.key, - C: make(chan bool), // close channel to signal data delivery - } - self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally) - glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", chunk.Key.Log(), len(b), off) +// implements the Joiner interface +func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader { - // waiting for the chunk retrieval - select { - case <-self.quitC: - // this is how we control process leakage (quitC is closed once join is finished (after timeout)) - // glog.V(logger.Detail).Infof("[BZZ] quit") - return - case <-chunk.C: // bells are ringing, data have been delivered - // glog.V(logger.Detail).Infof("[BZZ] chunk data received for %v", chunk.Key.Log()) + return &LazyChunkReader{ + key: key, + chunkC: chunkC, + chunkSize: self.chunkSize, + branches: self.branches, + hashSize: self.hashSize, } - if len(chunk.SData) == 0 { - // glog.V(logger.Detail).Infof("[BZZ] No payload in %v", chunk.Key.Log()) - return 0, notFound +} + +// Size is meant to be called on the LazySectionReader +func (self *LazyChunkReader) Size(quitC chan bool) (n int64, err error) { + if self.chunk != nil { + return self.chunk.Size, nil } - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - self.size = chunk.Size - if b == nil { + chunk := retrieve(self.key, self.chunkC, quitC) + if chunk == nil { + select { + case <-quitC: + return 0, errors.New("aborted") + default: + return 0, fmt.Errorf("root chunk not found for %v", self.key.Hex()) + } + } + self.chunk = chunk + return chunk.Size, nil +} + +// read at can be called numerous times +// concurrent reads are allowed +// Size() needs to be called synchronously on the LazyChunkReader first +func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { + // this is correct, a swarm doc cannot be zero length, so no EOF is expected + if len(b) == 0 { // glog.V(logger.Detail).Infof("[BZZ] Size query for %v", chunk.Key.Log()) - return + return 0, nil } - want := int64(len(b)) - if off+want > self.size { - want = self.size - off + quitC := make(chan bool) + size, err := self.Size(quitC) + if err != nil { + return 0, err } + glog.V(logger.Detail).Infof("readAt: len(b): %v, off: %v, size: %v ", len(b), off, size) + + errC := make(chan error) + // glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", self.chunk.Key.Log(), len(b), off) + + // } + // glog.V(logger.Detail).Infof("-> want: %v, off: %v size: %v ", want, off, self.size) var treeSize int64 var depth int // calculate depth and max treeSize - treeSize = self.chunker.chunkSize - for ; treeSize < chunk.Size; treeSize *= self.chunker.branches { + treeSize = self.chunkSize + for ; treeSize < size; treeSize *= self.branches { depth++ } wg := sync.WaitGroup{} wg.Add(1) - go self.join(b, off, off+want, depth, treeSize/self.chunker.branches, chunk, &wg) + go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC) go func() { wg.Wait() - close(self.errC) + close(errC) }() - select { - case err = <-self.errC: - // glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err) - read = len(b) - if off+int64(read) == self.size { - err = io.EOF - } - // glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err) - case <-self.quitC: - // glog.V(logger.Detail).Infof("[BZZ] ReadAt aborted at %d: %v", read, err) + + err = <-errC + if err != nil { + close(quitC) + + return 0, err } - return + // glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err) + glog.V(logger.Detail).Infof("end: len(b): %v, off: %v, size: %v ", len(b), off, size) + if off+int64(len(b)) >= size { + glog.V(logger.Detail).Infof(" len(b): %v EOF", len(b)) + return len(b), io.EOF + } + // glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err) + return len(b), nil } -func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup) { +func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { defer parentWg.Done() + // return NewDPA(&LocalStore{}) + glog.V(logger.Detail).Infof("inh len(b): %v, off: %v eoff: %v ", len(b), off, eoff) // glog.V(logger.Detail).Infof("[BZZ] depth: %v, loff: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, off, eoff, chunk.Size, treeSize) - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) // find appropriate block level for chunk.Size < treeSize && depth > 0 { - treeSize /= self.chunker.branches + treeSize /= self.branches depth-- } + // leaf chunk found if depth == 0 { - // glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize) - if int64(len(b)) != eoff-off { - //fmt.Printf("len(b) = %v off = %v eoff = %v", len(b), off, eoff) - panic("len(b) does not match") - } - + glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize) copy(b, chunk.SData[8+off:8+eoff]) return // simply give back the chunks reader for content chunks } - // subtree index + // subtree start := off / treeSize end := (eoff + treeSize - 1) / treeSize - wg := sync.WaitGroup{} + + wg := &sync.WaitGroup{} + defer wg.Wait() + glog.V(logger.Detail).Infof("[BZZ] start %v,end %v", start, end) for i := start; i < end; i++ { - soff := i * treeSize roff := soff seoff := soff + treeSize @@ -376,36 +400,91 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr if seoff > eoff { seoff = eoff } - + if depth > 1 { + wg.Wait() + } wg.Add(1) go func(j int64) { - childKey := chunk.SData[8+j*self.chunker.hashSize : 8+(j+1)*self.chunker.hashSize] - // glog.V(logger.Detail).Infof("[BZZ] subtree index: %v -> %v", j, childKey.Log()) - - ch := &Chunk{ - Key: childKey, - C: make(chan bool), // close channel to signal data delivery - } - // glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize) - self.chunkC <- ch // submit retrieval request, someone should be listening on the other side (or we will time out globally) - - // waiting for the chunk retrieval - select { - case <-self.quitC: - // this is how we control process leakage (quitC is closed once join is finished (after timeout)) + childKey := chunk.SData[8+j*self.hashSize : 8+(j+1)*self.hashSize] + // glog.V(logger.Detail).Infof("[BZZ] subtree ind.ex: %v -> %v", j, childKey.Log()) + chunk := retrieve(childKey, self.chunkC, quitC) + if chunk == nil { + select { + case errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize): + case <-quitC: + } return - case <-ch.C: // bells are ringing, data have been delivered - // glog.V(logger.Detail).Infof("[BZZ] chunk data received") } if soff < off { soff = off } - if len(ch.SData) == 0 { - self.errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize) - return - } - self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.chunker.branches, ch, &wg) + self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.branches, chunk, wg, errC, quitC) }(i) } //for - wg.Wait() +} + +// the helper method submits chunks for a key to a oueue (DPA) and +// block until they time out or arrive +// abort if quitC is readable +func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk { + chunk := &Chunk{ + Key: key, + C: make(chan bool), // close channel to signal data delivery + } + // glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize) + // submit chunk for retrieval + select { + case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally) + case <-quitC: + return nil + } + // waiting for the chunk retrieval + select { // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + + case <-quitC: + // this is how we control process leakage (quitC is closed once join is finished (after timeout)) + return nil + case <-chunk.C: // bells are ringing, data have been delivered + // glog.V(logger.Detail).Infof("[BZZ] chunk data received") + } + if len(chunk.SData) == 0 { + return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + + } + return chunk +} + +// Read keeps a cursor so cannot be called simulateously, see ReadAt +func (self *LazyChunkReader) Read(b []byte) (read int, err error) { + read, err = self.ReadAt(b, self.off) + glog.V(logger.Detail).Infof("[BZZ] read: %v, off: %v, error: %v", read, self.off, err) + + self.off += int64(read) + return +} + +// completely analogous to standard SectionReader implementation +var errWhence = errors.New("Seek: invalid whence") +var errOffset = errors.New("Seek: invalid offset") + +func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { + switch whence { + default: + return 0, errWhence + case 0: + offset += 0 + case 1: + offset += s.off + case 2: + if s.chunk == nil { + return 0, fmt.Errorf("seek from the end requires rootchunk for size. call Size first") + } + offset += s.chunk.Size + } + + if offset < 0 { + return 0, errOffset + } + s.off = offset + return offset, nil } diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 7dd301fff5..cdf549ff45 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -2,20 +2,33 @@ package storage import ( "bytes" - // "fmt" + "fmt" "io" + "runtime" + "sync" "testing" "time" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" ) +func init() { + glog.SetV(logger.Info) + glog.SetToStderr(true) +} + /* Tests TreeChunker by splitting and joining a random byte slice */ +type test interface { + Fatalf(string, ...interface{}) +} + type chunkerTester struct { - errors []error - chunks []*Chunk - timeout bool + chunks []*Chunk + t test } func (self *chunkerTester) checkChunks(t *testing.T, want int) { @@ -25,77 +38,70 @@ func (self *chunkerTester) checkChunks(t *testing.T, want int) { } } -func (self *chunkerTester) Split(chunker *TreeChunker, l int) (key Key, input []byte) { +func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup) (key Key) { // reset - self.errors = nil self.chunks = nil - self.timeout = false - - data, slice := testDataReader(l) - input = slice - key = make([]byte, 32) - chunkC := make(chan *Chunk, 1000) - errC := chunker.Split(key, data, chunkC, nil) quitC := make(chan bool) timeout := time.After(600 * time.Second) + if chunkC != nil { + go func() { + for { + select { + case <-timeout: + self.t.Fatalf("Join timeout error") - go func() { - LOOP: - for { - select { - case <-timeout: - self.timeout = true - break LOOP - - case chunk := <-chunkC: - if chunk != nil { + case chunk, ok := <-chunkC: + if !ok { + // glog.V(logger.Info).Infof("chunkC closed quitting") + close(quitC) + return + } + // glog.V(logger.Info).Infof("chunk %v received", len(self.chunks)) self.chunks = append(self.chunks, chunk) - } else { - break LOOP - } - - case err, ok := <-errC: - if err != nil { - self.errors = append(self.errors, err) - } - // fmt.Printf("err %v", err) - if !ok { - close(chunkC) - errC = nil + if chunk.wg != nil { + chunk.wg.Done() + } } } + }() + } + key, err := chunker.Split(data, size, chunkC, swg, nil) + if err != nil { + self.t.Fatalf("Split error: %v", err) + } + if chunkC != nil { + if swg != nil { + // glog.V(logger.Info).Infof("Waiting for storage to finish") + swg.Wait() + // glog.V(logger.Info).Infof("St orage finished") } - close(quitC) - }() - <-quitC // waiting for it to finish + close(chunkC) + } + if chunkC != nil { + <-quitC + } return } -func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionReader { +func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { // reset but not the chunks - self.errors = nil - self.timeout = false - chunkC := make(chan *Chunk, 1000) reader := chunker.Join(key, chunkC) - quitC := make(chan bool) timeout := time.After(600 * time.Second) i := 0 go func() { - LOOP: for { select { - case <-quitC: - break LOOP - case <-timeout: - self.timeout = true - break LOOP + self.t.Fatalf("Join timeout error") - case chunk := <-chunkC: + case chunk, ok := <-chunkC: + if !ok { + close(quitC) + return + } i++ - // dpaLogger.DebugDetailf("TESTER: chunk request %x", chunk.Key[:4]) // this just mocks the behaviour of a chunk store retrieval var found bool for _, ch := range self.chunks { @@ -106,56 +112,58 @@ func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionRea } } if !found { - // fmt.Printf("TESTER: chunk unknown for %x", chunk.Key[:4]) + self.t.Fatalf("not found ") } close(chunk.C) - // dpaLogger.DebugDetailf("TESTER: chunk request served %x", chunk.Key[:4]) } } }() return reader } -func testRandomData(chunker *TreeChunker, tester *chunkerTester, n int, chunks int, t *testing.T) { - key, input := tester.Split(chunker, n) +func testRandomData(n int, chunks int, t *testing.T) { + chunker := NewTreeChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data, input := testDataReaderAndSlice(n) - t.Logf(" Key = %x\n", key) + chunkC := make(chan *Chunk, 1000) + swg := &sync.WaitGroup{} - tester.checkChunks(t, chunks) - time.Sleep(100 * time.Millisecond) + splitter := chunker + key := tester.Split(splitter, data, int64(n), chunkC, swg) - reader := tester.Join(chunker, key, 0) + // t.Logf(" Key = %v\n", key) + + // tester.checkChunks(t, chunks) + chunkC = make(chan *Chunk, 1000) + quitC := make(chan bool) + + reader := tester.Join(chunker, key, 0, chunkC, quitC) output := make([]byte, n) r, err := reader.Read(output) if r != n || err != io.EOF { - t.Errorf("read error read: %v n = %v err = %v\n", r, n, err) + t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) } - // t.Logf(" IN: %x\nOUT: %x\n", input, output) - if !bytes.Equal(output, input) { - t.Errorf("input and output mismatch\n IN: %x\nOUT: %x\n", input, output) + if input != nil { + if !bytes.Equal(output, input) { + t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) + } } + close(chunkC) + <-quitC } func TestRandomData(t *testing.T) { - chunker, tester := chunkerAndTester() - testRandomData(chunker, tester, 60, 1, t) - testRandomData(chunker, tester, 179, 5, t) - testRandomData(chunker, tester, 253, 7, t) - // t.Logf("chunks %v", tester.chunks) + testRandomData(60, 1, t) + testRandomData(83, 3, t) + testRandomData(179, 5, t) + testRandomData(253, 7, t) } -func chunkerAndTester() (chunker *TreeChunker, tester *chunkerTester) { - chunker = NewTreeChunker(&ChunkerParams{ - Branches: 2, - Hash: "SHA256", - SplitTimeout: 10, - JoinTimeout: 10, - }) - tester = &chunkerTester{} - return -} - -func readAll(reader SectionReader, result []byte) { +func readAll(reader LazySectionReader, result []byte) { size := int64(len(result)) var end int64 @@ -169,46 +177,98 @@ func readAll(reader SectionReader, result []byte) { } } -func benchReadAll(reader SectionReader) { - size := reader.Size() +func benchReadAll(reader LazySectionReader) { + size, _ := reader.Size(nil) output := make([]byte, 1000) for pos := int64(0); pos < size; pos += 1000 { reader.ReadAt(output, pos) } } -func benchmarkJoinRandomData(n int, chunks int, t *testing.B) { - t.StopTimer() +func benchmarkJoin(n int, t *testing.B) { for i := 0; i < t.N; i++ { - // fmt.Printf("round %v\n", i) - chunker, tester := chunkerAndTester() - key, _ := tester.Split(chunker, n) - // fmt.Printf("split done %v, joining...\n", i) + chunker := NewTreeChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data := testDataReader(n) + + chunkC := make(chan *Chunk, 1000) + swg := &sync.WaitGroup{} + + key := tester.Split(chunker, data, int64(n), chunkC, swg) t.StartTimer() - reader := tester.Join(chunker, key, i) - // fmt.Printf("join done %v, reading...\n", i) + chunkC = make(chan *Chunk, 1000) + quitC := make(chan bool) + reader := tester.Join(chunker, key, i, chunkC, quitC) + t.StopTimer() benchReadAll(reader) + close(chunkC) + <-quitC } } -func benchmarkSplitRandomData(n int, chunks int, t *testing.B) { +func benchmarkSplitTree(n int, t *testing.B) { + t.ReportAllocs() for i := 0; i < t.N; i++ { - chunker, tester := chunkerAndTester() - tester.Split(chunker, n) + chunker := NewTreeChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data := testDataReader(n) + // glog.V(logger.Info).Infof("splitting data of length %v", n) + tester.Split(chunker, data, int64(n), nil, nil) } + stats := new(runtime.MemStats) + runtime.ReadMemStats(stats) + fmt.Println(stats.Sys) } -func BenchmarkJoinRandomData_100_2(t *testing.B) { benchmarkJoinRandomData(100, 3, t) } -func BenchmarkJoinRandomData_1000_2(t *testing.B) { benchmarkJoinRandomData(1000, 3, t) } -func BenchmarkJoinRandomData_10000_2(t *testing.B) { benchmarkJoinRandomData(10000, 3, t) } -func BenchmarkJoinRandomData_100000_2(t *testing.B) { benchmarkJoinRandomData(100000, 3, t) } -func BenchmarkJoinRandomData_1000000_2(t *testing.B) { benchmarkJoinRandomData(1000000, 3, t) } +func benchmarkSplitPyramid(n int, t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + splitter := NewPyramidChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data := testDataReader(n) + // glog.V(logger.Info).Infof("splitting data of length %v", n) + tester.Split(splitter, data, int64(n), nil, nil) + } + stats := new(runtime.MemStats) + runtime.ReadMemStats(stats) + fmt.Println(stats.Sys) +} -func BenchmarkSplitRandomData_100_2(t *testing.B) { benchmarkSplitRandomData(100, 3, t) } -func BenchmarkSplitRandomData_1000_2(t *testing.B) { benchmarkSplitRandomData(1000, 3, t) } -func BenchmarkSplitRandomData_10000_2(t *testing.B) { benchmarkSplitRandomData(10000, 3, t) } -func BenchmarkSplitRandomData_100000_2(t *testing.B) { benchmarkSplitRandomData(100000, 3, t) } -func BenchmarkSplitRandomData_1000000_2(t *testing.B) { benchmarkSplitRandomData(1000000, 3, t) } -func BenchmarkSplitRandomData_10000000_2(t *testing.B) { benchmarkSplitRandomData(10000000, 3, t) } +func BenchmarkJoin_100_2(t *testing.B) { benchmarkJoin(100, t) } +func BenchmarkJoin_1000_2(t *testing.B) { benchmarkJoin(1000, t) } +func BenchmarkJoin_10000_2(t *testing.B) { benchmarkJoin(10000, t) } +func BenchmarkJoin_100000_2(t *testing.B) { benchmarkJoin(100000, t) } +func BenchmarkJoin_1000000_2(t *testing.B) { benchmarkJoin(1000000, t) } -// go test -bench ./bzz -cpuprofile cpu.out -memprofile mem.out +func BenchmarkSplitTree_2(t *testing.B) { benchmarkSplitTree(100, t) } +func BenchmarkSplitTree_2h(t *testing.B) { benchmarkSplitTree(500, t) } +func BenchmarkSplitTree_3(t *testing.B) { benchmarkSplitTree(1000, t) } +func BenchmarkSplitTree_3h(t *testing.B) { benchmarkSplitTree(5000, t) } +func BenchmarkSplitTree_4(t *testing.B) { benchmarkSplitTree(10000, t) } +func BenchmarkSplitTree_4h(t *testing.B) { benchmarkSplitTree(50000, t) } +func BenchmarkSplitTree_5(t *testing.B) { benchmarkSplitTree(100000, t) } +func BenchmarkSplitTree_6(t *testing.B) { benchmarkSplitTree(1000000, t) } +func BenchmarkSplitTree_7(t *testing.B) { benchmarkSplitTree(10000000, t) } +func BenchmarkSplitTree_8(t *testing.B) { benchmarkSplitTree(100000000, t) } + +func BenchmarkSplitPyramid_2(t *testing.B) { benchmarkSplitPyramid(100, t) } +func BenchmarkSplitPyramid_2h(t *testing.B) { benchmarkSplitPyramid(500, t) } +func BenchmarkSplitPyramid_3(t *testing.B) { benchmarkSplitPyramid(1000, t) } +func BenchmarkSplitPyramid_3h(t *testing.B) { benchmarkSplitPyramid(5000, t) } +func BenchmarkSplitPyramid_4(t *testing.B) { benchmarkSplitPyramid(10000, t) } +func BenchmarkSplitPyramid_4h(t *testing.B) { benchmarkSplitPyramid(50000, t) } +func BenchmarkSplitPyramid_5(t *testing.B) { benchmarkSplitPyramid(100000, t) } +func BenchmarkSplitPyramid_6(t *testing.B) { benchmarkSplitPyramid(1000000, t) } +func BenchmarkSplitPyramid_7(t *testing.B) { benchmarkSplitPyramid(10000000, t) } +func BenchmarkSplitPyramid_8(t *testing.B) { benchmarkSplitPyramid(100000000, t) } + +// godep go test -bench ./swarm/storage -cpuprofile cpu.out -memprofile mem.out diff --git a/swarm/storage/chunkreader.go b/swarm/storage/chunkreader.go deleted file mode 100644 index b147c85bf8..0000000000 --- a/swarm/storage/chunkreader.go +++ /dev/null @@ -1,194 +0,0 @@ -package storage - -import ( - "bytes" - "errors" - "io" -) - -type Bounded interface { - Size() int64 -} - -type Sliced interface { - Slice(int64, int64) (b []byte, err error) -} - -// Size, Seek, Read, ReadAt -type SectionReader interface { - Bounded - io.Seeker - io.Reader - io.ReaderAt -} - -// ChunkReader implements SectionReader on a section -// of an underlying ReaderAt. -type ChunkReader struct { - r io.ReaderAt - base int64 - off int64 - limit int64 -} - -// NewChunkReader returns a ChunkReader that reads from r -// starting at offset off and stops with EOF after n bytes. -func NewChunkReader(r io.ReaderAt, off int64, n int64) *ChunkReader { - return &ChunkReader{r: r, base: off, off: off, limit: off + n} -} - -// ByteSliceReader just extends byte.Reader to make base slice accessible -type ByteSliceReader struct { - *bytes.Reader - base []byte -} - -func NewByteSliceReader(b []byte) *ByteSliceReader { - return &ByteSliceReader{ - base: b, - Reader: bytes.NewReader(b), - } -} - -// ByteSliceReader implements the Sliced interface -func (self *ByteSliceReader) Slice(from, to int64) (b []byte, err error) { - if from < 0 || to >= int64(self.Len()) { - err = io.EOF - } else { - b = self.base[from:to] - } - return -} - -// NewChunkReaderFromBytes is a convenience shortcut to get a SectionReader over a byte slice -func NewChunkReaderFromBytes(b []byte) *ChunkReader { - return NewChunkReader(NewByteSliceReader(b), 0, int64(len(b))) -} - -/* -The following is adapted from io.SectionReader -*/ - -func (s *ChunkReader) Size() int64 { - return s.limit - s.base -} - -var errWhence = errors.New("Seek: invalid whence") -var errOffset = errors.New("Seek: invalid offset") - -func (s *ChunkReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - default: - return 0, errWhence - case 0: - offset += s.base - case 1: - offset += s.off - case 2: - offset += s.limit - } - if offset < s.base { - return 0, errOffset - } - s.off = offset - return offset - s.base, nil -} - -func (s *ChunkReader) Read(p []byte) (n int, err error) { - if s.off >= s.limit { - return 0, io.EOF - } - if max := s.limit - s.off; int64(len(p)) > max { - p = p[0:max] - } - n, err = s.r.ReadAt(p, s.off) - s.off += int64(n) - return -} - -func (s *ChunkReader) ReadAt(p []byte, off int64) (n int, err error) { - if off < 0 || off >= s.limit-s.base { - return 0, io.EOF - } - off += s.base - if max := s.limit - off; int64(len(p)) > max { - p = p[0:max] - n, err = s.r.ReadAt(p, off) - if err == nil { - err = io.EOF - } - return n, err - } - n, err = s.r.ReadAt(p, off) - return -} - -// added methods to that ChunkReader implements the Sliced interface -func (s *ChunkReader) Slice(from, to int64) (b []byte, err error) { - if from < 0 || to >= s.Size() { - err = io.EOF - } else { - if sl, ok := s.r.(Sliced); ok { - b, err = sl.Slice(s.base+from, s.base+to) - } else { - err = errors.New("not sliceable base") - } - } - return -} - -// added method so that ChunkReader implements the io.WriterTo interface -// WriteTo method is used by io.Copy -// This is so that we avoid one extra step of allocation (if the underlying initial Reader implements Sliced -func (r *ChunkReader) WriteTo(w io.Writer) (n int64, err error) { - var b []byte - var m int - // if b, _ := r.Slice(r.off-r.base, r.limit-r.base); b == nil { - // if slices not available we do it with extra allocation - b = make([]byte, r.limit-r.off) - m, err = r.Read(b) - if err != nil { - return - } - // } - m, err = w.Write(b) - if m > len(b) { - panic("bytes.Reader.WriteTo: invalid Write count") - } - r.off = r.base + int64(m) - n = int64(m) - if m != len(b) && err == nil { - err = io.ErrShortWrite - } - // w - return -} - -func (self *LazyChunkReader) Size() (n int64) { - self.ReadAt(nil, 0) - return self.size -} - -func (self *LazyChunkReader) Read(b []byte) (read int, err error) { - read, err = self.ReadAt(b, self.off) - self.off += int64(read) - return -} - -func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - default: - return 0, errWhence - case 0: - offset += 0 - case 1: - offset += s.off - case 2: - offset += s.size - } - if offset < 0 { - return 0, errOffset - } - s.off = offset - return offset, nil -} diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 40dc35fc69..55fcbfd409 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -1,6 +1,7 @@ package storage import ( + "bytes" "crypto/rand" "io" "sync" @@ -10,61 +11,39 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" ) -func testDataReader(l int) (r *ChunkReader, slice []byte) { +func testDataReader(l int) (r io.Reader) { + return io.LimitReader(rand.Reader, int64(l)) +} + +func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) { slice = make([]byte, l) if _, err := rand.Read(slice); err != nil { panic("rand error") } - r = NewChunkReaderFromBytes(slice) - return -} - -func randomChunks(l int64, branches int64, chunkC chan *Chunk) (key Key, errC chan error) { - chunker := NewTreeChunker(&ChunkerParams{ - Branches: branches, - Hash: defaultHash, - SplitTimeout: splitTimeout, - }) - key = make([]byte, 32) - b := make([]byte, l) - _, err := rand.Read(b) - if err != nil { - panic("no rand") - } - wg := &sync.WaitGroup{} - errC = chunker.Split(key, NewChunkReaderFromBytes(b), chunkC, wg) - wg.Wait() + r = bytes.NewReader(slice) return } func testStore(m ChunkStore, l int64, branches int64, t *testing.T) { chunkC := make(chan *Chunk) - key, errC := randomChunks(l, branches, chunkC) - -SPLIT: - for { - select { - case chunk := <-chunkC: + go func() { + for chunk := range chunkC { m.Put(chunk) - case err, ok := <-errC: - if err != nil { - t.Errorf("Chunker error: %v", err) - return - } - if !ok { - break SPLIT + if chunk.wg != nil { + chunk.wg.Done() } } - } + }() chunker := NewTreeChunker(&ChunkerParams{ - Branches: branches, - Hash: defaultHash, - SplitTimeout: splitTimeout, + Branches: branches, + Hash: defaultHash, }) + swg := &sync.WaitGroup{} + key, err := chunker.Split(rand.Reader, l, chunkC, swg, nil) + swg.Wait() + close(chunkC) chunkC = make(chan *Chunk) - var r SectionReader - r = chunker.Join(key, chunkC) quit := make(chan bool) @@ -73,22 +52,26 @@ SPLIT: go func(chunk *Chunk) { storedChunk, err := m.Get(chunk.Key) if err == notFound { - glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key) + glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log()) } else if err != nil { - glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %x: %v", chunk.Key, err) + glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %v: %v", chunk.Key.Log(), err) } else { chunk.SData = storedChunk.SData + chunk.Size = storedChunk.Size } - glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key[:4]) + glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log()) close(chunk.C) }(ch) } + close(quit) }() + r := chunker.Join(key, chunkC) b := make([]byte, l) n, err := r.ReadAt(b, 0) if err != io.EOF { - t.Errorf("read error (%v/%v) %v", n, l, err) - close(quit) + t.Fatalf("read error (%v/%v) %v", n, l, err) } + close(chunkC) + <-quit } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 922ff3bec2..34a4639a6d 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -2,6 +2,7 @@ package storage import ( "errors" + "io" "sync" "time" @@ -72,33 +73,14 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA { // FS-aware API and httpaccess // Chunk retrieval blocks on netStore requests with a timeout so reader will // report error if retrieval of chunks within requested range time out. -func (self *DPA) Retrieve(key Key) SectionReader { +func (self *DPA) Retrieve(key Key) LazySectionReader { return self.Chunker.Join(key, self.retrieveC) } // Public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess -func (self *DPA) Store(data SectionReader, wg *sync.WaitGroup) (key Key, err error) { - key = make([]byte, self.Chunker.KeySize()) - errC := self.Chunker.Split(key, data, self.storeC, wg) - -SPLIT: - for { - select { - case err, ok := <-errC: - if err != nil { - glog.V(logger.Error).Infof("[BZZ] chunker split error: %v", err) - } - if !ok { - break SPLIT - } - - case <-self.quitC: - break SPLIT - } - } - return - +func (self *DPA) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key Key, err error) { + return self.Chunker.Split(data, size, self.storeC, nil, wg) } func (self *DPA) Start() { @@ -164,7 +146,7 @@ func (self *DPA) storeLoop() { go func(chunk *Chunk) { self.Put(chunk) if chunk.wg != nil { - glog.V(logger.Detail).Infof("[BZZ] DPA.storeLoop %v", chunk.Key.Log()) + glog.V(logger.Detail).Infof("[BZZ] dpa: store loop %v", chunk.Key.Log()) chunk.wg.Done() } }(ch) diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index a4400783fb..4c50a7214f 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -29,9 +29,9 @@ func TestDPArandom(t *testing.T) { ChunkStore: localStore, } dpa.Start() - reader, slice := testDataReader(testDataSize) + reader, slice := testDataReaderAndSlice(testDataSize) wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, wg) + key, err := dpa.Store(reader, testDataSize, wg) if err != nil { t.Errorf("Store error: %v", err) } @@ -85,9 +85,9 @@ func TestDPA_capacity(t *testing.T) { ChunkStore: localStore, } dpa.Start() - reader, slice := testDataReader(testDataSize) + reader, slice := testDataReaderAndSlice(testDataSize) wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, wg) + key, err := dpa.Store(reader, testDataSize, wg) if err != nil { t.Errorf("Store error: %v", err) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 81d4a6f2d0..68487368f1 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -1,5 +1,9 @@ package storage +import ( + "encoding/binary" +) + // LocalStore is a combination of inmemory db over a disk persisted db // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores type LocalStore struct { @@ -48,6 +52,7 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { if err != nil { return } + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) self.memStore.Put(chunk) return } diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index f415bdfa59..c5e0f6227f 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -3,7 +3,6 @@ package storage import ( - "bytes" "sync" ) @@ -44,41 +43,6 @@ func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { return } -func (x Key) Size() uint { - return uint(len(x)) -} - -func (x Key) isEqual(y Key) bool { - return bytes.Compare(x, y) == 0 -} - -func (h Key) bits(i, j uint) uint { - ii := i >> 3 - jj := i & 7 - if ii >= h.Size() { - return 0 - } - - if jj+j <= 8 { - return uint((h[ii] >> jj) & ((1 << j) - 1)) - } - - res := uint(h[ii] >> jj) - jj = 8 - jj - j -= jj - for j != 0 { - ii++ - if j < 8 { - res += uint(h[ii]&((1< task %v (%v)", index, n) + select { + case tasks <- &Task{Index: int64(index), Data: buffer[:n+8], Last: last}: + case <-abortC: + return nil, err + } + if last { + // glog.V(logger.Info).Infof("last task %v (%v)", index, n) + break + } + } + // Wait for the workers and return + close(tasks) + pend.Wait() + + // glog.V(logger.Info).Infof("len: %v", results.Levels[0][0]) + key := results.Levels[0][0].Children[0][:] + return key, nil +} + +func (self *PyramidChunker) processor(pend *sync.WaitGroup, tasks chan *Task, results *Tree) { + defer pend.Done() + + // glog.V(logger.Info).Infof("processor started") + // Start processing leaf chunks ad infinitum + hasher := self.hashFunc() + for task := range tasks { + depth, pow := len(results.Levels)-1, self.branches + // glog.V(logger.Info).Infof("task: %v, last: %v", task.Index, task.Last) + + var node *Node + for depth >= 0 { + // New chunk received, reset the hasher and start processing + hasher.Reset() + + if node == nil { // Leaf node, hash the data chunk + hasher.Write(task.Data) + } else { // Internal node, hash the children + for _, hash := range node.Children { + hasher.Write(hash[:]) + } + } + hash := hasher.Sum(nil) + last := task.Last || (node != nil) && node.Last + // Insert the subresult into the memoization tree + results.Lock.Lock() + if node = results.Levels[depth][task.Index/pow]; node == nil { + // Figure out the pending tasks + pending := self.branches + if task.Index/pow == results.Chunks/pow { + pending = (results.Chunks + pow/self.branches - 1) / (pow / self.branches) % self.branches + } + node = &Node{pending, make([]common.Hash, pending), last} + results.Levels[depth][task.Index/pow] = node + } + node.Pending-- + i := task.Index / (pow / self.branches) % self.branches + if last { + node.Pending -= self.branches - i + node.Children = node.Children[:i+1] + node.Last = true + } + copy(node.Children[i][:], hash) + left := node.Pending + + if depth+1 < len(results.Levels) { + delete(results.Levels[depth+1], task.Index/(pow/self.branches)) + } + results.Lock.Unlock() + // If there's more work to be done, leave for others + // glog.V(logger.Info).Infof("left %v", left) + if left > 0 { + break + } + // We're the last ones in this batch, merge the children together + depth-- + pow *= self.branches + } + pend.Done() + } +} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 124a56a085..8d7e7fdd38 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -5,6 +5,7 @@ import ( "crypto" "fmt" "hash" + "io" "sync" "github.com/ethereum/go-ethereum/common" @@ -13,10 +14,47 @@ import ( type Hasher func() hash.Hash +// Peer is the recorded as Source on the chunk +// should probably not be here? but network should wrap chunk object type Peer interface{} type Key []byte +func (x Key) Size() uint { + return uint(len(x)) +} + +func (x Key) isEqual(y Key) bool { + return bytes.Compare(x, y) == 0 +} + +func (h Key) bits(i, j uint) uint { + ii := i >> 3 + jj := i & 7 + if ii >= h.Size() { + return 0 + } + + if jj+j <= 8 { + return uint((h[ii] >> jj) & ((1 << j) - 1)) + } + + res := uint(h[ii] >> jj) + jj = 8 - jj + j -= jj + for j != 0 { + ii++ + if j < 8 { + res += uint(h[ii]&((1< Swarm Domain Name Registrar") + self.dns = ens.NewENS(transactOpts, config.EnsRoot, self.backend) + glog.V(logger.Debug).Infof("[BZZ] -> Swarm Domain Name Registrar @ address %v", config.EnsRoot) self.api = api.NewApi(self.dpa, self.dns) // Manifests for Smart Hosting diff --git a/swarm/test/connections/00.sh b/swarm/test/connections/00.sh index 7f4c4445e9..b080d51cfe 100644 --- a/swarm/test/connections/00.sh +++ b/swarm/test/connections/00.sh @@ -1,33 +1,25 @@ #!/bin/bash -dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh - swarm init 4 echo "expect each node to have 3 peers" -cmd="'net.peerCount'" +cmd="net.peerCount" sleep 5 -swarm attach 00 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 00 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 01 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 02 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 03 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" swarm stop all -echo "after static nodes is deleted, connections are recovered from kaddb in bzz-peers.json" -# echo rm -rf $DATA_ROOT/enodes\* -# echo rm -rf $DATA_ROOT/data/\*/static-nodes.json -rm -rf $DATA_ROOT/enodes* -rm -rf $DATA_ROOT/data/*/static-nodes.json +echo "connections are recovered from kaddb in bzz-peers.json" swarm cluster 4 echo "expect each node to have 3 peers" -cmd="'net.peerCount'" -sleep 10 -swarm attach 00 --exec "$cmd" |tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +sleep 5 +swarm execute 00 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 01 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 02 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 03 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" swarm stop all diff --git a/swarm/test/swap/00.sh b/swarm/test/swap/00.sh index 0cb0b05a4d..81006a8d6f 100644 --- a/swarm/test/swap/00.sh +++ b/swarm/test/swap/00.sh @@ -4,7 +4,7 @@ echo " two nodes that do not sync and do not have any funds" echo " cannot retrieve content from each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh FILE_00=/tmp/1K.0 randomfile 1 > $FILE_00 diff --git a/swarm/test/swap/01.sh b/swarm/test/swap/01.sh index 362f0da73a..d9d50fa2cc 100644 --- a/swarm/test/swap/01.sh +++ b/swarm/test/swap/01.sh @@ -3,19 +3,20 @@ echo " two nodes that do not sync but have enough funds" echo " can retrieve content from each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/..s/test.sh file=/tmp/test.file mininginterval=120 key=/tmp/key -logargs="--verbosity=0 --vmodule='swarm/*=6'" -# logargs='--verbosity=6' +# logargs="--verbosity=0 --vmodule='swarm/*=6'" +logargs='--verbosity=6' +# swarm init 2 --mine --bzznosync --bzznoswap=false $logargsc +# echo "Mining some ether..." +# sleep $mininginterval -swarm init 2 --mine --bzznosync $logargs +swarm cluster 2 --mine --bzznosync --bzznoswap=false $logargsc -echo "Mining some ether..." -sleep $mininginterval randomfile 10 > $file swarm up 00 $file|tail -n1 > $key diff --git a/swarm/test/syncing/00.sh b/swarm/test/syncing/00.sh index a793fea916..3785ba1e2f 100644 --- a/swarm/test/syncing/00.sh +++ b/swarm/test/syncing/00.sh @@ -4,7 +4,7 @@ echo " two nodes that sync (no swap and do not have any funds)" echo " can be in sync content with each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh mkdir -p /tmp/swarm-test-files FILE_00=/tmp/swarm-test-files/00 @@ -28,7 +28,6 @@ swarm needs 00 $key $FILE_00 swarm needs 01 $key $FILE_00 swarm stop 01 -# exit 1; swarm up 00 $FILE_01|tail -n1 > $key swarm needs 00 $key $FILE_01 @@ -46,7 +45,8 @@ swarm needs 00 $key $FILE_03 swarm stop 00 swarm up 01 $FILE_04|tail -n1 > $key swarm needs 01 $key $FILE_04 -swarm start 00 #--bzznoswap +swarm start 00 +sleep $wait swarm needs 00 $key $FILE_04 swarm stop all diff --git a/swarm/test/syncing/01.sh b/swarm/test/syncing/01.sh index 4e152dcd9f..bd090d391a 100644 --- a/swarm/test/syncing/01.sh +++ b/swarm/test/syncing/01.sh @@ -4,7 +4,7 @@ echo " two nodes that do not have any funds" echo " can still sync content with each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh key=/tmp/key long=/tmp/10M diff --git a/swarm/test/syncing/02.sh b/swarm/test/syncing/02.sh index 48760d6e97..2b0d23443c 100644 --- a/swarm/test/syncing/02.sh +++ b/swarm/test/syncing/02.sh @@ -5,26 +5,26 @@ echo " two nodes that sync (no swap and do not have any funds)" echo " can sync content with each other even with intermittent network connection" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh long=/tmp/10M key=/tmp/key -randomfile 10000 > $long +randomfile 100000 > $long ls -l $long -swarm init 2 -sleep $wait +swarm init 2 --vmodule='swarm/*=5' swarm up 00 $long |tail -n1 > $key & -sleep $wait -swarm attach 01 -exec "'bzz.blockNetworkRead(true)'" -sleep $wait -swarm attach 01 -exec "'bzz.blockNetworkRead(false)'" -sleep $wait -swarm attach 01 -exec "'bzz.blockNetworkRead(true)'" -sleep $wait +sleep 1 +swarm execute 01 'bzz.blockNetworkRead(true)' +sleep 3 +swarm execute 01 'bzz.blockNetworkRead(false)' +# sleep $wait +# swarm attach 01 -exec "'bzz.blockNetworkRead(true)'" +# sleep $wait swarm stop 01 -swarm start 01 -swarm needs 01 $key $long - +# swarm start 01 +# sleep $wait +# swarm needs 01 $key $long +# sleep 3 swarm stop all \ No newline at end of file diff --git a/swarm/test/test.sh b/swarm/test/test.sh new file mode 100644 index 0000000000..01703452a5 --- /dev/null +++ b/swarm/test/test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +TEST_DIR=`dirname $0` +TEST_NAME=`basename $0 .sh` +TEST_TYPE=`basename $TEST_DIR` +export IP_ADDR="[::]" + + +export SWARM_NETWORK_ID=322$TEST_NAME +export SWARM_DIR=~/bzz/test/$TEST_TYPE + +rm -rf $SWARM_DIR/$SWARM_NETWORK_ID + +wait=1 + + + +function randomfile { + dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null +} \ No newline at end of file