diff --git a/README.md b/README.md index 2fabcdd794..86fcdeee4d 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm | Command | Description | |:----------:|-------------| -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. | | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). | diff --git a/accounts/keystore/file_cache.go b/accounts/keystore/file_cache.go index 26da34dcfd..73ff6ae9ee 100644 --- a/accounts/keystore/file_cache.go +++ b/accounts/keystore/file_cache.go @@ -56,9 +56,9 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er var newLastMod time.Time for _, fi := range files { - // Skip any non-key files from the folder path := filepath.Join(keyDir, fi.Name()) - if skipKeyFile(fi) { + // Skip any non-key files from the folder + if nonKeyFile(fi) { log.Trace("Ignoring file on account scan", "path", path) continue } @@ -88,8 +88,8 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er return creates, deletes, updates, nil } -// skipKeyFile ignores editor backups, hidden files and folders/symlinks. -func skipKeyFile(fi os.FileInfo) bool { +// nonKeyFile ignores editor backups, hidden files and folders/symlinks. +func nonKeyFile(fi os.FileInfo) bool { // Skip editor backups and UNIX-style hidden files. if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") { return true diff --git a/build/ci.go b/build/ci.go index b3a986dca9..ff23e15fdb 100644 --- a/build/ci.go +++ b/build/ci.go @@ -147,6 +147,9 @@ var ( debEthereum, } + // Packages to be cross-compiled by the xgo command + allCrossCompiledArchiveFiles = append(allToolsArchiveFiles, swarmArchiveFiles...) + // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: wily is unsupported because it was officially deprecated on lanchpad. @@ -1009,7 +1012,7 @@ func doXgo(cmdline []string) { if *alltools { args = append(args, []string{"--dest", GOBIN}...) - for _, res := range allToolsArchiveFiles { + for _, res := range allCrossCompiledArchiveFiles { if strings.HasPrefix(res, GOBIN) { // Binary tool found, cross build it explicitly args = append(args, "./"+filepath.Join("cmd", filepath.Base(res))) diff --git a/cmd/evm/compiler.go b/cmd/evm/compiler.go index f8aef2074c..c019a2fe70 100644 --- a/cmd/evm/compiler.go +++ b/cmd/evm/compiler.go @@ -50,6 +50,6 @@ func compileCmd(ctx *cli.Context) error { if err != nil { return err } - fmt.Fprintln(ctx.App.Writer, bin) + fmt.Println(bin) return nil } diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go index 0f3491818d..4a442cf784 100644 --- a/cmd/evm/disasm.go +++ b/cmd/evm/disasm.go @@ -45,6 +45,6 @@ func disasmCmd(ctx *cli.Context) error { } code := strings.TrimSpace(string(in[:])) - fmt.Fprintf(ctx.App.Writer, "%v\n", code) + fmt.Printf("%v\n", code) return asm.PrintDisassembled(code) } diff --git a/cmd/evm/internal/compiler/compiler.go b/cmd/evm/internal/compiler/compiler.go index a2abc8f7da..753ca62264 100644 --- a/cmd/evm/internal/compiler/compiler.go +++ b/cmd/evm/internal/compiler/compiler.go @@ -30,11 +30,10 @@ func Compile(fn string, src []byte, debug bool) (string, error) { bin, compileErrors := compiler.Compile() if len(compileErrors) > 0 { // report errors - errs := "" for _, err := range compileErrors { - errs += fmt.Sprintf("%s:%v\n", fn, err) + fmt.Printf("%s:%v\n", fn, err) } - return "", errors.New(errs + "compiling failed\n") + return "", errors.New("compiling failed") } return bin, nil } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index fed41f53a1..7138a9ddd4 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -128,13 +128,13 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalString(CodeFileFlag.Name) == "-" { //Try reading from stdin if hexcode, err = ioutil.ReadAll(os.Stdin); err != nil { - fmt.Fprintf(ctx.App.ErrWriter, "Could not load code from stdin: %v\n", err) + fmt.Printf("Could not load code from stdin: %v\n", err) os.Exit(1) } } else { // Codefile with hex assembly if hexcode, err = ioutil.ReadFile(ctx.GlobalString(CodeFileFlag.Name)); err != nil { - fmt.Fprintf(ctx.App.ErrWriter, "Could not load code from file: %v\n", err) + fmt.Printf("Could not load code from file: %v\n", err) os.Exit(1) } } @@ -172,11 +172,11 @@ func runCmd(ctx *cli.Context) error { if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" { f, err := os.Create(cpuProfilePath) if err != nil { - fmt.Fprintf(ctx.App.ErrWriter, "could not create CPU profile: %v\n", err) + fmt.Println("could not create CPU profile: ", err) os.Exit(1) } if err := pprof.StartCPUProfile(f); err != nil { - fmt.Fprintf(ctx.App.ErrWriter, "could not start CPU profile: %v\n", err) + fmt.Println("could not start CPU profile: ", err) os.Exit(1) } defer pprof.StopCPUProfile() @@ -200,17 +200,17 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.IntermediateRoot(true) - fmt.Fprintln(ctx.App.Writer, string(statedb.Dump())) + fmt.Println(string(statedb.Dump())) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { f, err := os.Create(memProfilePath) if err != nil { - fmt.Fprintf(ctx.App.ErrWriter, "could not create memory profile: %v\n", err) + fmt.Println("could not create memory profile: ", err) os.Exit(1) } if err := pprof.WriteHeapProfile(f); err != nil { - fmt.Fprintf(ctx.App.ErrWriter, "could not create memory profile: %v\n", err) + fmt.Println("could not write memory profile: ", err) os.Exit(1) } f.Close() @@ -218,17 +218,17 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DebugFlag.Name) { if debugLogger != nil { - fmt.Fprintln(ctx.App.ErrWriter, "#### TRACE ####") - vm.WriteTrace(ctx.App.ErrWriter, debugLogger.StructLogs()) + fmt.Fprintln(os.Stderr, "#### TRACE ####") + vm.WriteTrace(os.Stderr, debugLogger.StructLogs()) } - fmt.Fprintln(ctx.App.ErrWriter, "#### LOGS ####") - vm.WriteLogs(ctx.App.ErrWriter, statedb.Logs()) + fmt.Fprintln(os.Stderr, "#### LOGS ####") + vm.WriteLogs(os.Stderr, statedb.Logs()) } if ctx.GlobalBool(StatDumpFlag.Name) { var mem goruntime.MemStats goruntime.ReadMemStats(&mem) - fmt.Fprintf(ctx.App.ErrWriter, `evm execution time: %v + fmt.Fprintf(os.Stderr, `evm execution time: %v heap objects: %d allocations: %d total allocations: %d @@ -238,9 +238,9 @@ Gas used: %d `, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas) } if tracer == nil { - fmt.Fprintf(ctx.App.Writer, "0x%x\n", ret) + fmt.Printf("0x%x\n", ret) if err != nil { - fmt.Fprintf(ctx.App.ErrWriter, " error: %v\n", err) + fmt.Printf(" error: %v\n", err) } } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index bee1b34dcc..6d5ff069f6 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -107,7 +107,7 @@ func stateTestCmd(ctx *cli.Context) error { } // print state root for evmlab tracing (already committed above, so no need to delete objects again if ctx.GlobalBool(MachineFlag.Name) && state != nil { - fmt.Fprintf(ctx.App.ErrWriter, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) } results = append(results, *result) @@ -115,13 +115,13 @@ func stateTestCmd(ctx *cli.Context) error { // Print any structured logs collected if ctx.GlobalBool(DebugFlag.Name) { if debugger != nil { - fmt.Fprintln(ctx.App.ErrWriter, "#### TRACE ####") - vm.WriteTrace(ctx.App.ErrWriter, debugger.StructLogs()) + fmt.Fprintln(os.Stderr, "#### TRACE ####") + vm.WriteTrace(os.Stderr, debugger.StructLogs()) } } } } out, _ := json.MarshalIndent(results, "", " ") - fmt.Fprintln(ctx.App.Writer, string(out)) + fmt.Println(string(out)) return nil } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 8d8b10f8fa..34ba877020 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -31,7 +31,7 @@ import ( ) const ( - ipcAPIs = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0" + ipcAPIs = "admin:1.0 debug:1.0 eth:1.0 ethash:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0" httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0" ) diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index a517623381..d22bd81103 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -678,9 +678,9 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da // Build and deploy the dashboard service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // dashboardInfos is returned from a dashboard status check to allow reporting diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index 1a7b484267..a7d99a2979 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -100,9 +100,9 @@ func deployEthstats(client *sshClient, network string, port int, secret string, // Build and deploy the ethstats service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // ethstatsInfos is returned from an ethstats status check to allow reporting diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index bb43e5fe4e..e916deaf6f 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -38,7 +38,7 @@ ADD chain.json /chain.json RUN \ echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' > explorer.sh && \ echo '(cd ../etherchain-light && npm start &)' >> explorer.sh && \ - echo '/parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh + echo 'exec /parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh ENTRYPOINT ["/bin/sh", "explorer.sh"] ` @@ -140,9 +140,9 @@ func deployExplorer(client *sshClient, network string, chainspec []byte, config // Build and deploy the boot or seal node service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // explorerInfos is returned from a block explorer status check to allow reporting diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 8365bf47d0..06c9fc0f58 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -133,9 +133,9 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config // Build and deploy the faucet service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // faucetInfos is returned from a faucet status check to allow reporting various diff --git a/cmd/puppeth/module_nginx.go b/cmd/puppeth/module_nginx.go index 35c0efc8ad..7f87661d3b 100644 --- a/cmd/puppeth/module_nginx.go +++ b/cmd/puppeth/module_nginx.go @@ -81,9 +81,9 @@ func deployNginx(client *sshClient, network string, port int, nocache bool) ([]b // Build and deploy the reverse-proxy service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // nginxInfos is returned from an nginx reverse-proxy status check to allow diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 34377733f3..a480a894eb 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -139,9 +139,9 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n // Build and deploy the boot or seal node service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // nodeInfos is returned from a boot or seal node status check to allow reporting diff --git a/cmd/puppeth/module_wallet.go b/cmd/puppeth/module_wallet.go index 5e5032bedf..90812c4a0b 100644 --- a/cmd/puppeth/module_wallet.go +++ b/cmd/puppeth/module_wallet.go @@ -37,7 +37,7 @@ ADD genesis.json /genesis.json RUN \ echo 'node server.js &' > wallet.sh && \ echo 'geth --cache 512 init /genesis.json' >> wallet.sh && \ - echo $'geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh + echo $'exec geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh RUN \ sed -i 's/PuppethNetworkID/{{.NetworkID}}/g' dist/js/etherwallet-master.js && \ @@ -120,9 +120,9 @@ func deployWallet(client *sshClient, network string, bootnodes []string, config // Build and deploy the boot or seal node service if nocache { - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network)) } - return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network)) + return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network)) } // walletInfos is returned from a web wallet status check to allow reporting diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index ce2acdcc97..cda8c41c32 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -68,6 +68,7 @@ const ( SWARM_ENV_SWAP_API = "SWARM_SWAP_API" SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE" SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY" + SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE" SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK" SWARM_ENV_ENS_API = "SWARM_ENS_API" SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" @@ -131,7 +132,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) { log.Debug(printConfig(config)) } -//override the current config with whatever is in the config file, if a config file has been provided +//configFileOverride overrides the current config with the config file, if a config file has been provided func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) { var err error @@ -141,7 +142,8 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" { utils.Fatalf("Config file flag provided with invalid file path") } - f, err := os.Open(filepath) + var f *os.File + f, err = os.Open(filepath) if err != nil { return nil, err } @@ -204,6 +206,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.SyncUpdateDelay = d } + if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) { + currentConfig.LightNodeEnabled = true + } + if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) { currentConfig.DeliverySkipCheck = true } @@ -301,6 +307,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } } + if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" { + if lightnode, err := strconv.ParseBool(lne); err != nil { + currentConfig.LightNodeEnabled = lightnode + } + } + if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" { currentConfig.SwapAPI = swapapi } diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index d5011e3a70..02198f878e 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "os" "os/exec" "testing" @@ -559,3 +560,16 @@ func TestValidateConfig(t *testing.T) { } } } + +func assignTCPPort() (string, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + l.Close() + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return "", err + } + return port, nil +} diff --git a/cmd/swarm/fs.go b/cmd/swarm/fs.go index 0124586cfe..3dc38ca4da 100644 --- a/cmd/swarm/fs.go +++ b/cmd/swarm/fs.go @@ -92,7 +92,7 @@ func listMounts(cliContext *cli.Context) { mf := []fuse.MountInfo{} err = client.CallContext(ctx, &mf, "swarmfs_listmounts") if err != nil { - utils.Fatalf("encountered an error calling the RPC endpoint while unmounting: %v", err) + utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err) } if len(mf) == 0 { fmt.Print("Could not found any swarmfs mounts. Please make sure you've specified the correct RPC endpoint\n") diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index 0cbf0eb137..4f38b094bb 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// +build linux darwin freebsd +// +build linux freebsd package main @@ -43,6 +43,11 @@ type testFile struct { } // TestCLISwarmFs is a high-level test of swarmfs +// +// This test fails on travis for macOS as this executable exits with code 1 +// and without any log messages in the log: +// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse. +// This is the reason for this file not being built on darwin architecture. func TestCLISwarmFs(t *testing.T) { cluster := newTestCluster(t, 3) defer cluster.Shutdown() diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 9185af980f..ac09ae9981 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -123,6 +123,11 @@ var ( Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, } + SwarmLightNodeEnabled = cli.BoolFlag{ + Name: "lightnode", + Usage: "Enable Swarm LightNode (default false)", + EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE, + } SwarmDeliverySkipCheckFlag = cli.BoolFlag{ Name: "delivery-skip-check", Usage: "Skip chunk delivery check (default false)", @@ -317,23 +322,23 @@ Downloads a swarm bzz uri to the given dir. When no dir is provided, working dir Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", Subcommands: []cli.Command{ { - Action: add, + Action: manifestAdd, CustomHelpTemplate: helpTemplate, Name: "add", Usage: "add a new path to the manifest", - ArgsUsage: " []", + ArgsUsage: " ", Description: "Adds a new path to the manifest", }, { - Action: update, + Action: manifestUpdate, CustomHelpTemplate: helpTemplate, Name: "update", Usage: "update the hash for an already existing path in the manifest", - ArgsUsage: " []", + ArgsUsage: " ", Description: "Update the hash for an already existing path in the manifest", }, { - Action: remove, + Action: manifestRemove, CustomHelpTemplate: helpTemplate, Name: "remove", Usage: "removes a path from the manifest", @@ -464,6 +469,7 @@ pv(1) tool to get a progress bar: SwarmSwapAPIFlag, SwarmSyncDisabledFlag, SwarmSyncUpdateDelay, + SwarmLightNodeEnabled, SwarmDeliverySkipCheckFlag, SwarmListenAddrFlag, SwarmPortFlag, diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 82166edf6c..0216ffc1dd 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -18,10 +18,8 @@ package main import ( - "encoding/json" "fmt" - "mime" - "path/filepath" + "os" "strings" "github.com/ethereum/go-ethereum/cmd/utils" @@ -30,127 +28,118 @@ import ( "gopkg.in/urfave/cli.v1" ) -const bzzManifestJSON = "application/bzz-manifest+json" - -func add(ctx *cli.Context) { +// manifestAdd adds a new entry to the manifest at the given path. +// New entry hash, the last argument, must be the hash of a manifest +// with only one entry, which meta-data will be added to the original manifest. +// On success, this function will print new (updated) manifest's hash. +func manifestAdd(ctx *cli.Context) { args := ctx.Args() - if len(args) < 3 { - utils.Fatalf("Need at least three arguments []") + if len(args) != 3 { + utils.Fatalf("Need exactly three arguments ") } var ( mhash = args[0] path = args[1] hash = args[2] - - ctype string - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - if len(args) > 3 { - ctype = args[3] - } else { - ctype = mime.TypeByExtension(filepath.Ext(path)) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + + m, _, err := client.DownloadManifest(hash) + if err != nil { + utils.Fatalf("Error downloading manifest to add: %v", err) + } + l := len(m.Entries) + if l == 0 { + utils.Fatalf("No entries in manifest %s", hash) + } else if l > 1 { + utils.Fatalf("Too many entries in manifest %s", hash) } - newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype) + newManifest := addEntryToManifest(client, mhash, path, m.Entries[0]) fmt.Println(newManifest) - - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } } -func update(ctx *cli.Context) { - +// manifestUpdate replaces an existing entry of the manifest at the given path. +// New entry hash, the last argument, must be the hash of a manifest +// with only one entry, which meta-data will be added to the original manifest. +// On success, this function will print hash of the updated manifest. +func manifestUpdate(ctx *cli.Context) { args := ctx.Args() - if len(args) < 3 { - utils.Fatalf("Need at least three arguments ") + if len(args) != 3 { + utils.Fatalf("Need exactly three arguments ") } var ( mhash = args[0] path = args[1] hash = args[2] - - ctype string - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - if len(args) > 3 { - ctype = args[3] - } else { - ctype = mime.TypeByExtension(filepath.Ext(path)) + + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + + m, _, err := client.DownloadManifest(hash) + if err != nil { + utils.Fatalf("Error downloading manifest to update: %v", err) + } + l := len(m.Entries) + if l == 0 { + utils.Fatalf("No entries in manifest %s", hash) + } else if l > 1 { + utils.Fatalf("Too many entries in manifest %s", hash) } - newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype) + newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true) + if defaultEntryUpdated { + // Print informational message to stderr + // allowing the user to get the new manifest hash from stdout + // without the need to parse the complete output. + fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too") + } fmt.Println(newManifest) - - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } } -func remove(ctx *cli.Context) { +// manifestRemove removes an existing entry of the manifest at the given path. +// On success, this function will print hash of the manifest which does not +// contain the path. +func manifestRemove(ctx *cli.Context) { args := ctx.Args() - if len(args) < 2 { - utils.Fatalf("Need at least two arguments ") + if len(args) != 2 { + utils.Fatalf("Need exactly two arguments ") } var ( mhash = args[0] path = args[1] - - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - newManifest := removeEntryFromManifest(ctx, mhash, path) - fmt.Println(newManifest) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } + newManifest := removeEntryFromManifest(client, mhash, path) + fmt.Println(newManifest) } -func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string { - - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) - longestPathEntry = api.ManifestEntry{} - ) +func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string { + var longestPathEntry = api.ManifestEntry{} mroot, isEncrypted, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } - //TODO: check if the "hash" to add is valid and present in swarm - _, _, err = client.DownloadManifest(hash) - if err != nil { - utils.Fatalf("Hash to add is not present: %v", err) - } - // See if we path is in this Manifest or do we have to dig deeper - for _, entry := range mroot.Entries { - if path == entry.Path { + for _, e := range mroot.Entries { + if path == e.Path { utils.Fatalf("Path %s already present, not adding anything", path) } else { - if entry.ContentType == bzzManifestJSON { - prfxlen := strings.HasPrefix(path, entry.Path) + if e.ContentType == api.ManifestType { + prfxlen := strings.HasPrefix(path, e.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { - longestPathEntry = entry + longestPathEntry = e } } } @@ -159,25 +148,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin if longestPathEntry.Path != "" { // Load the child Manifest add the entry there newPath := path[len(longestPathEntry.Path):] - newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) + newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if longestPathEntry.Path == entry.Path { - entry.Hash = newHash + for _, e := range mroot.Entries { + if longestPathEntry.Path == e.Path { + e.Hash = newHash } - newMRoot.Entries = append(newMRoot.Entries, entry) + newMRoot.Entries = append(newMRoot.Entries, e) } mroot = newMRoot } else { // Add the entry in the leaf Manifest - newEntry := api.ManifestEntry{ - Hash: hash, - Path: path, - ContentType: ctype, - } - mroot.Entries = append(mroot.Entries, newEntry) + entry.Path = path + mroot.Entries = append(mroot.Entries, entry) } newManifestHash, err := client.UploadManifest(mroot, isEncrypted) @@ -185,14 +170,16 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin utils.Fatalf("Manifest upload failed: %v", err) } return newManifestHash - } -func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string { - +// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash +// finding the path recursively through all nested manifests. Argument isRoot is used for default +// entry update detection. If the updated entry has the same hash as the default entry, then the +// default entry in root manifest will be updated too. +// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and +// a a bool that is true if default entry is updated. +func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) { var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) newEntry = api.ManifestEntry{} longestPathEntry = api.ManifestEntry{} ) @@ -202,17 +189,18 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st utils.Fatalf("Manifest download failed: %v", err) } - //TODO: check if the "hash" with which to update is valid and present in swarm - // See if we path is in this Manifest or do we have to dig deeper - for _, entry := range mroot.Entries { - if path == entry.Path { - newEntry = entry + for _, e := range mroot.Entries { + if path == e.Path { + newEntry = e + // keep the reference of the hash of the entry that should be replaced + // for default entry detection + oldHash = e.Hash } else { - if entry.ContentType == bzzManifestJSON { - prfxlen := strings.HasPrefix(path, entry.Path) + if e.ContentType == api.ManifestType { + prfxlen := strings.HasPrefix(path, e.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { - longestPathEntry = entry + longestPathEntry = e } } } @@ -225,50 +213,50 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st if longestPathEntry.Path != "" { // Load the child Manifest add the entry there newPath := path[len(longestPathEntry.Path):] - newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) + var newHash string + newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if longestPathEntry.Path == entry.Path { - entry.Hash = newHash + for _, e := range mroot.Entries { + if longestPathEntry.Path == e.Path { + e.Hash = newHash } - newMRoot.Entries = append(newMRoot.Entries, entry) + newMRoot.Entries = append(newMRoot.Entries, e) } mroot = newMRoot } - if newEntry.Path != "" { + // update the manifest if the new entry is found and + // check if default entry should be updated + if newEntry.Path != "" || isRoot { // Replace the hash for leaf Manifest newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if newEntry.Path == entry.Path { - myEntry := api.ManifestEntry{ - Hash: hash, - Path: entry.Path, - ContentType: ctype, - } - newMRoot.Entries = append(newMRoot.Entries, myEntry) - } else { + for _, e := range mroot.Entries { + if newEntry.Path == e.Path { + entry.Path = e.Path newMRoot.Entries = append(newMRoot.Entries, entry) + } else if isRoot && e.Path == "" && e.Hash == oldHash { + entry.Path = e.Path + newMRoot.Entries = append(newMRoot.Entries, entry) + defaultEntryUpdated = true + } else { + newMRoot.Entries = append(newMRoot.Entries, e) } } mroot = newMRoot } - newManifestHash, err := client.UploadManifest(mroot, isEncrypted) + newManifestHash, err = client.UploadManifest(mroot, isEncrypted) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } - return newManifestHash + return newManifestHash, oldHash, defaultEntryUpdated } -func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { - +func removeEntryFromManifest(client *swarm.Client, mhash, path string) string { var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) entryToRemove = api.ManifestEntry{} longestPathEntry = api.ManifestEntry{} ) @@ -283,7 +271,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if path == entry.Path { entryToRemove = entry } else { - if entry.ContentType == bzzManifestJSON { + if entry.ContentType == api.ManifestType { prfxlen := strings.HasPrefix(path, entry.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { longestPathEntry = entry @@ -299,7 +287,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if longestPathEntry.Path != "" { // Load the child Manifest remove the entry there newPath := path[len(longestPathEntry.Path):] - newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath) + newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go new file mode 100644 index 0000000000..08fe0b2eb7 --- /dev/null +++ b/cmd/swarm/manifest_test.go @@ -0,0 +1,579 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/swarm/api" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" +) + +// TestManifestChange tests manifest add, update and remove +// cli commands without encryption. +func TestManifestChange(t *testing.T) { + testManifestChange(t, false) +} + +// TestManifestChange tests manifest add, update and remove +// cli commands with encryption enabled. +func TestManifestChangeEncrypted(t *testing.T) { + testManifestChange(t, true) +} + +// testManifestChange performs cli commands: +// - manifest add +// - manifest update +// - manifest remove +// on a manifest, testing the functionality of this +// comands on paths that are in root manifest or a nested one. +// Argument encrypt controls whether to use encryption or not. +func testManifestChange(t *testing.T, encrypt bool) { + t.Parallel() + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + origDir := filepath.Join(tmp, "orig") + if err := os.Mkdir(origDir, 0777); err != nil { + t.Fatal(err) + } + + indexDataFilename := filepath.Join(origDir, "index.html") + err = ioutil.WriteFile(indexDataFilename, []byte("

Test

"), 0666) + if err != nil { + t.Fatal(err) + } + // Files paths robots.txt and robots.html share the same prefix "robots." + // which will result a manifest with a nested manifest under path "robots.". + // This will allow testing manifest changes on both root and nested manifest. + err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("No Robots Allowed"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + indexDataFilename, + "up", + origDir, + } + if encrypt { + args = append(args, "--encrypt") + } + + origManifestHash := runSwarmExpectHash(t, args...) + + checkHashLength(t, origManifestHash, encrypt) + + client := swarm.NewClient(cluster.Nodes[0].URL) + + // upload a new file and use its manifest to add it the original manifest. + t.Run("add", func(t *testing.T) { + humansData := []byte("Ann\nBob") + humansDataFilename := filepath.Join(tmp, "humans.txt") + err = ioutil.WriteFile(humansDataFilename, humansData, 0666) + if err != nil { + t.Fatal(err) + } + + humansManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + humansDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "add", + origManifestHash, + "humans.txt", + humansManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "humans.txt" { + found = true + if e.Size != int64(len(humansData)) { + t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for humans.txt") + } + ct := "text/plain; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no humans.txt in new manifest") + } + + checkFile(t, client, newManifestHash, "humans.txt", humansData) + }) + + // upload a new file and use its manifest to add it the original manifest, + // but ensure that the file will be in the nested manifest of the original one. + t.Run("add nested", func(t *testing.T) { + robotsData := []byte(`{"disallow": "/"}`) + robotsDataFilename := filepath.Join(tmp, "robots.json") + err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666) + if err != nil { + t.Fatal(err) + } + + robotsManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + robotsDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "add", + origManifestHash, + "robots.json", + robotsManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "json" { + found = true + if e.Size != int64(len(robotsData)) { + t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for robots.json") + } + ct := "application/json" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break loop + } + } + } + } + if !found { + t.Fatal("no robots.json in new manifest") + } + + checkFile(t, client, newManifestHash, "robots.json", robotsData) + }) + + // upload a new file and use its manifest to change the file it the original manifest. + t.Run("update", func(t *testing.T) { + indexData := []byte("

Ethereum Swarm

") + indexDataFilename := filepath.Join(tmp, "index.html") + err = ioutil.WriteFile(indexDataFilename, indexData, 0666) + if err != nil { + t.Fatal(err) + } + + indexManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + indexDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "index.html", + indexManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "index.html" { + found = true + if e.Size != int64(len(indexData)) { + t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for index.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no index.html in new manifest") + } + + checkFile(t, client, newManifestHash, "index.html", indexData) + + // check default entry change + checkFile(t, client, newManifestHash, "", indexData) + }) + + // upload a new file and use its manifest to change the file it the original manifest, + // but ensure that the file is in the nested manifest of the original one. + t.Run("update nested", func(t *testing.T) { + robotsData := []byte(`Only humans allowed!!!`) + robotsDataFilename := filepath.Join(tmp, "robots.html") + err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666) + if err != nil { + t.Fatal(err) + } + + humansManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + robotsDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "robots.html", + humansManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "html" { + found = true + if e.Size != int64(len(robotsData)) { + t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for robots.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break loop + } + } + } + } + if !found { + t.Fatal("no robots.html in new manifest") + } + + checkFile(t, client, newManifestHash, "robots.html", robotsData) + }) + + // remove a file from the manifest. + t.Run("remove", func(t *testing.T) { + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "remove", + origManifestHash, + "mutants.txt", + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "mutants.txt" { + found = true + break + } + } + if found { + t.Fatal("mutants.txt is not removed") + } + }) + + // remove a file from the manifest, but ensure that the file is in + // the nested manifest of the original one. + t.Run("remove nested", func(t *testing.T) { + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "remove", + origManifestHash, + "robots.html", + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "html" { + found = true + break loop + } + } + } + } + if found { + t.Fatal("robots.html in not removed") + } + }) +} + +// TestNestedDefaultEntryUpdate tests if the default entry is updated +// if the file in nested manifest used for it is also updated. +func TestNestedDefaultEntryUpdate(t *testing.T) { + testNestedDefaultEntryUpdate(t, false) +} + +// TestNestedDefaultEntryUpdateEncrypted tests if the default entry +// of encrypted upload is updated if the file in nested manifest +// used for it is also updated. +func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { + testNestedDefaultEntryUpdate(t, true) +} + +func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { + t.Parallel() + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + origDir := filepath.Join(tmp, "orig") + if err := os.Mkdir(origDir, 0777); err != nil { + t.Fatal(err) + } + + indexData := []byte("

Test

") + indexDataFilename := filepath.Join(origDir, "index.html") + err = ioutil.WriteFile(indexDataFilename, indexData, 0666) + if err != nil { + t.Fatal(err) + } + // Add another file with common prefix as the default entry to test updates of + // default entry with nested manifests. + err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + indexDataFilename, + "up", + origDir, + } + if encrypt { + args = append(args, "--encrypt") + } + + origManifestHash := runSwarmExpectHash(t, args...) + + checkHashLength(t, origManifestHash, encrypt) + + client := swarm.NewClient(cluster.Nodes[0].URL) + + newIndexData := []byte("

Ethereum Swarm

") + newIndexDataFilename := filepath.Join(tmp, "index.html") + err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666) + if err != nil { + t.Fatal(err) + } + + newIndexManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + newIndexDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "index.html", + newIndexManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "index." { + found = true + newManifest = downloadManifest(t, client, e.Hash, encrypt) + break + } + } + if !found { + t.Fatal("no index. path in new manifest") + } + + found = false + for _, e := range newManifest.Entries { + if e.Path == "html" { + found = true + if e.Size != int64(len(newIndexData)) { + t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for index.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no html in new manifest") + } + + checkFile(t, client, newManifestHash, "index.html", newIndexData) + + // check default entry change + checkFile(t, client, newManifestHash, "", newIndexData) +} + +func runSwarmExpectHash(t *testing.T, args ...string) (hash string) { + t.Helper() + hashRegexp := `[a-f\d]{64,128}` + up := runSwarm(t, args...) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + return matches[0] +} + +func checkHashLength(t *testing.T, hash string, encrypted bool) { + t.Helper() + l := len(hash) + if encrypted && l != 128 { + t.Errorf("expected hash length 128, got %v", l) + } + if !encrypted && l != 64 { + t.Errorf("expected hash length 64, got %v", l) + } +} + +func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) { + t.Helper() + m, isEncrypted, err := client.DownloadManifest(hash) + if err != nil { + t.Fatal(err) + } + + if encrypted != isEncrypted { + t.Error("new manifest encryption flag is not correct") + } + return m +} + +func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) { + t.Helper() + f, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + + got, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, expected) { + t.Errorf("expected file content %q, got %q", expected, got) + } +} diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index a70c4686dd..90d3c98ba6 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -17,12 +17,15 @@ package main import ( + "context" "fmt" "io/ioutil" "net" "os" "path/filepath" "runtime" + "sync" + "syscall" "testing" "time" @@ -218,14 +221,12 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { } // assign ports - httpPort, err := assignTCPPort() - if err != nil { - t.Fatal(err) - } - p2pPort, err := assignTCPPort() + ports, err := getAvailableTCPPorts(2) if err != nil { t.Fatal(err) } + p2pPort := ports[0] + httpPort := ports[1] // start the node node.Cmd = runSwarm(t, @@ -246,6 +247,17 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { } }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // ensure that all ports have active listeners + // so that the next node will not get the same + // when calling getAvailableTCPPorts + err = waitTCPPorts(ctx, ports...) + if err != nil { + t.Fatal(err) + } + // wait for the node to start for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { node.Client, err = rpc.Dial(conf.IPCEndpoint()) @@ -280,14 +292,12 @@ func newTestNode(t *testing.T, dir string) *testNode { node := &testNode{Dir: dir} // assign ports - httpPort, err := assignTCPPort() - if err != nil { - t.Fatal(err) - } - p2pPort, err := assignTCPPort() + ports, err := getAvailableTCPPorts(2) if err != nil { t.Fatal(err) } + p2pPort := ports[0] + httpPort := ports[1] // start the node node.Cmd = runSwarm(t, @@ -308,6 +318,17 @@ func newTestNode(t *testing.T, dir string) *testNode { } }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // ensure that all ports have active listeners + // so that the next node will not get the same + // when calling getAvailableTCPPorts + err = waitTCPPorts(ctx, ports...) + if err != nil { + t.Fatal(err) + } + // wait for the node to start for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) { node.Client, err = rpc.Dial(conf.IPCEndpoint()) @@ -343,15 +364,92 @@ func (n *testNode) Shutdown() { } } -func assignTCPPort() (string, error) { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return "", err +// getAvailableTCPPorts returns a set of ports that +// nothing is listening on at the time. +// +// Function assignTCPPort cannot be called in sequence +// and guardantee that the same port will be returned in +// different calls as the listener is closed within the function, +// not after all listeners are started and selected unique +// available ports. +func getAvailableTCPPorts(count int) (ports []string, err error) { + for i := 0; i < count; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + // defer close in the loop to be sure the same port will not + // be selected in the next iteration + defer l.Close() + + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + return nil, err + } + ports = append(ports, port) + } + return ports, nil +} + +// waitTCPPorts blocks until tcp connections can be +// established on all provided ports. It runs all +// ports dialers in parallel, and returns the first +// encountered error. +// See waitTCPPort also. +func waitTCPPorts(ctx context.Context, ports ...string) error { + var err error + // mu locks err variable that is assigned in + // other goroutines + var mu sync.Mutex + + // cancel is canceling all goroutines + // when the firs error is returned + // to prevent unnecessary waiting + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var wg sync.WaitGroup + for _, port := range ports { + wg.Add(1) + go func(port string) { + defer wg.Done() + + e := waitTCPPort(ctx, port) + + mu.Lock() + defer mu.Unlock() + if e != nil && err == nil { + err = e + cancel() + } + }(port) + } + wg.Wait() + + return err +} + +// waitTCPPort blocks until tcp connection can be established +// ona provided port. It has a 3 minute timeout as maximum, +// to prevent long waiting, but it can be shortened with +// a provided context instance. Dialer has a 10 second timeout +// in every iteration, and connection refused error will be +// retried in 100 milliseconds periods. +func waitTCPPort(ctx context.Context, port string) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + + for { + c, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, "tcp", "127.0.0.1:"+port) + if err != nil { + if operr, ok := err.(*net.OpError); ok { + if syserr, ok := operr.Err.(*os.SyscallError); ok && syserr.Err == syscall.ECONNREFUSED { + time.Sleep(100 * time.Millisecond) + continue + } + } + return err + } + return c.Close() } - l.Close() - _, port, err := net.SplitHostPort(l.Addr().String()) - if err != nil { - return "", err - } - return port, nil } diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 8ba0e7c5f0..9eae2a3f80 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -98,6 +98,17 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } + if defaultPath != "" { + // construct absolute default path + absDefaultPath, _ := filepath.Abs(defaultPath) + absFile, _ := filepath.Abs(file) + // make sure absolute directory ends with only one "/" + // to trim it from absolute default path and get relative default path + absFile = strings.TrimRight(absFile, "/") + "/" + if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) { + defaultPath = strings.TrimPrefix(absDefaultPath, absFile) + } + } return client.UploadDirectory(file, defaultPath, "", toEncrypt) } } else { diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 2afc9b3a11..c3199dadc6 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -273,3 +273,84 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } } } + +// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute +// default paths and with encryption. +func TestCLISwarmUpDefaultPath(t *testing.T) { + testCLISwarmUpDefaultPath(false, false, t) + testCLISwarmUpDefaultPath(false, true, t) + testCLISwarmUpDefaultPath(true, false, t) + testCLISwarmUpDefaultPath(true, true, t) +} + +func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("

Test

"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666) + if err != nil { + t.Fatal(err) + } + + defaultPath := "index.html" + if absDefaultPath { + defaultPath = filepath.Join(tmp, defaultPath) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + defaultPath, + "up", + tmp, + } + if toEncrypt { + args = append(args, "--encrypt") + } + + up := runSwarm(t, args...) + hashRegexp := `[a-f\d]{64,128}` + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + hash := matches[0] + + client := swarm.NewClient(cluster.Nodes[0].URL) + + m, isEncrypted, err := client.DownloadManifest(hash) + if err != nil { + t.Fatal(err) + } + + if toEncrypt != isEncrypted { + t.Error("downloaded manifest is not encrypted") + } + + var found bool + var entriesCount int + for _, e := range m.Entries { + entriesCount++ + if e.Path == "" { + found = true + } + } + + if !found { + t.Error("manifest default entry was not found") + } + + if entriesCount != 3 { + t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3) + } +} diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 8968f500f1..59bb3d40b4 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -672,6 +672,11 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int { return new(big.Int).Set(diffNoTurn) } +// Close implements consensus.Engine. It's a noop for clique as there is are no background threads. +func (c *Clique) Close() error { + return nil +} + // APIs implements consensus.Engine, returning the user facing RPC API to allow // controlling the signer voting. func (c *Clique) APIs(chain consensus.ChainReader) []rpc.API { diff --git a/consensus/consensus.go b/consensus/consensus.go index 5774af1a78..8271754445 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -96,6 +96,9 @@ type Engine interface { // APIs returns the RPC APIs this consensus engine provides. APIs(chain ChainReader) []rpc.API + + // Close terminates any background threads maintained by the consensus engine. + Close() error } // PoW is a consensus engine based on proof-of-work. diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index f0c6465fdb..e7625f7c00 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -730,6 +730,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { go func(idx int) { defer pend.Done() ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}) + defer ethash.Close() if err := ethash.VerifySeal(nil, block.Header()); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) } diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go new file mode 100644 index 0000000000..a04ea235d9 --- /dev/null +++ b/consensus/ethash/api.go @@ -0,0 +1,117 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethash + +import ( + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +var errEthashStopped = errors.New("ethash stopped") + +// API exposes ethash related methods for the RPC interface. +type API struct { + ethash *Ethash // Make sure the mode of ethash is normal. +} + +// GetWork returns a work package for external miner. +// +// The work package consists of 3 strings: +// result[0] - 32 bytes hex encoded current block header pow-hash +// result[1] - 32 bytes hex encoded seed hash used for DAG +// result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty +func (api *API) GetWork() ([3]string, error) { + if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { + return [3]string{}, errors.New("not supported") + } + + var ( + workCh = make(chan [3]string, 1) + errc = make(chan error, 1) + ) + + select { + case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}: + case <-api.ethash.exitCh: + return [3]string{}, errEthashStopped + } + + select { + case work := <-workCh: + return work, nil + case err := <-errc: + return [3]string{}, err + } +} + +// SubmitWork can be used by external miner to submit their POW solution. +// It returns an indication if the work was accepted. +// Note either an invalid solution, a stale work a non-existent work will return false. +func (api *API) SubmitWork(nonce types.BlockNonce, hash, digest common.Hash) bool { + if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { + return false + } + + var errc = make(chan error, 1) + + select { + case api.ethash.submitWorkCh <- &mineResult{ + nonce: nonce, + mixDigest: digest, + hash: hash, + errc: errc, + }: + case <-api.ethash.exitCh: + return false + } + + err := <-errc + return err == nil +} + +// SubmitHashrate can be used for remote miners to submit their hash rate. +// This enables the node to report the combined hash rate of all miners +// which submit work through this node. +// +// It accepts the miner hash rate and an identifier which must be unique +// between nodes. +func (api *API) SubmitHashRate(rate hexutil.Uint64, id common.Hash) bool { + if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { + return false + } + + var done = make(chan struct{}, 1) + + select { + case api.ethash.submitRateCh <- &hashrate{done: done, rate: uint64(rate), id: id}: + case <-api.ethash.exitCh: + return false + } + + // Block until hash rate submitted successfully. + <-done + + return true +} + +// GetHashrate returns the current hashrate for local CPU miner and remote miner. +func (api *API) GetHashrate() uint64 { + return uint64(api.ethash.Hashrate()) +} diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index f79dd6c36b..0cb3059b9d 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -33,7 +33,9 @@ import ( "unsafe" mmap "github.com/edsrzf/mmap-go" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rpc" @@ -389,6 +391,30 @@ type Config struct { PowMode Mode } +// mineResult wraps the pow solution parameters for the specified block. +type mineResult struct { + nonce types.BlockNonce + mixDigest common.Hash + hash common.Hash + + errc chan error +} + +// hashrate wraps the hash rate submitted by the remote sealer. +type hashrate struct { + id common.Hash + ping time.Time + rate uint64 + + done chan struct{} +} + +// sealWork wraps a seal work package for remote sealer. +type sealWork struct { + errc chan error + res chan [3]string +} + // Ethash is a consensus engine based on proof-of-work implementing the ethash // algorithm. type Ethash struct { @@ -403,15 +429,25 @@ type Ethash struct { update chan struct{} // Notification channel to update mining parameters hashrate metrics.Meter // Meter tracking the average hashrate + // Remote sealer related fields + workCh chan *types.Block // Notification channel to push new work to remote sealer + resultCh chan *types.Block // Channel used by mining threads to return result + fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work + submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result + fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. + submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate + // The fields below are hooks for testing shared *Ethash // Shared PoW verifier to avoid cache regeneration fakeFail uint64 // Block number which fails PoW check even in fake mode fakeDelay time.Duration // Time delay to sleep for before returning from verify - lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields + lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields + closeOnce sync.Once // Ensures exit channel will not be closed twice. + exitCh chan chan error // Notification channel to exiting backend threads } -// New creates a full sized ethash PoW scheme. +// New creates a full sized ethash PoW scheme and starts a background thread for remote mining. func New(config Config) *Ethash { if config.CachesInMem <= 0 { log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) @@ -423,19 +459,43 @@ func New(config Config) *Ethash { if config.DatasetDir != "" && config.DatasetsOnDisk > 0 { log.Info("Disk storage enabled for ethash DAGs", "dir", config.DatasetDir, "count", config.DatasetsOnDisk) } - return &Ethash{ - config: config, - caches: newlru("cache", config.CachesInMem, newCache), - datasets: newlru("dataset", config.DatasetsInMem, newDataset), - update: make(chan struct{}), - hashrate: metrics.NewMeter(), + ethash := &Ethash{ + config: config, + caches: newlru("cache", config.CachesInMem, newCache), + datasets: newlru("dataset", config.DatasetsInMem, newDataset), + update: make(chan struct{}), + hashrate: metrics.NewMeter(), + workCh: make(chan *types.Block), + resultCh: make(chan *types.Block), + fetchWorkCh: make(chan *sealWork), + submitWorkCh: make(chan *mineResult), + fetchRateCh: make(chan chan uint64), + submitRateCh: make(chan *hashrate), + exitCh: make(chan chan error), } + go ethash.remote() + return ethash } // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. func NewTester() *Ethash { - return New(Config{CachesInMem: 1, PowMode: ModeTest}) + ethash := &Ethash{ + config: Config{PowMode: ModeTest}, + caches: newlru("cache", 1, newCache), + datasets: newlru("dataset", 1, newDataset), + update: make(chan struct{}), + hashrate: metrics.NewMeter(), + workCh: make(chan *types.Block), + resultCh: make(chan *types.Block), + fetchWorkCh: make(chan *sealWork), + submitWorkCh: make(chan *mineResult), + fetchRateCh: make(chan chan uint64), + submitRateCh: make(chan *hashrate), + exitCh: make(chan chan error), + } + go ethash.remote() + return ethash } // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts @@ -489,6 +549,22 @@ func NewShared() *Ethash { return &Ethash{shared: sharedEthash} } +// Close closes the exit channel to notify all backend threads exiting. +func (ethash *Ethash) Close() error { + var err error + ethash.closeOnce.Do(func() { + // Short circuit if the exit channel is not allocated. + if ethash.exitCh == nil { + return + } + errc := make(chan error) + ethash.exitCh <- errc + err = <-errc + close(ethash.exitCh) + }) + return err +} + // cache tries to retrieve a verification cache for the specified block number // by first checking against a list of in-memory caches, then against caches // stored on disk, and finally generating one if none can be found. @@ -561,14 +637,44 @@ func (ethash *Ethash) SetThreads(threads int) { // Hashrate implements PoW, returning the measured rate of the search invocations // per second over the last minute. +// Note the returned hashrate includes local hashrate, but also includes the total +// hashrate of all remote miner. func (ethash *Ethash) Hashrate() float64 { - return ethash.hashrate.Rate1() + // Short circuit if we are run the ethash in normal/test mode. + if ethash.config.PowMode != ModeNormal && ethash.config.PowMode != ModeTest { + return ethash.hashrate.Rate1() + } + var res = make(chan uint64, 1) + + select { + case ethash.fetchRateCh <- res: + case <-ethash.exitCh: + // Return local hashrate only if ethash is stopped. + return ethash.hashrate.Rate1() + } + + // Gather total submitted hash rate of remote sealers. + return ethash.hashrate.Rate1() + float64(<-res) } -// APIs implements consensus.Engine, returning the user facing RPC APIs. Currently -// that is empty. +// APIs implements consensus.Engine, returning the user facing RPC APIs. func (ethash *Ethash) APIs(chain consensus.ChainReader) []rpc.API { - return nil + // In order to ensure backward compatibility, we exposes ethash RPC APIs + // to both eth and ethash namespaces. + return []rpc.API{ + { + Namespace: "eth", + Version: "1.0", + Service: &API{ethash}, + Public: true, + }, + { + Namespace: "ethash", + Version: "1.0", + Service: &API{ethash}, + Public: true, + }, + } } // SeedHash is the seed to use for generating a verification cache and the mining diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 31116da437..ccdd30fb0f 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -23,7 +23,10 @@ import ( "os" "sync" "testing" + "time" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) @@ -32,6 +35,7 @@ func TestTestMode(t *testing.T) { head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} ethash := NewTester() + defer ethash.Close() block, err := ethash.Seal(nil, types.NewBlockWithHeader(head), nil) if err != nil { t.Fatalf("failed to seal block: %v", err) @@ -52,6 +56,7 @@ func TestCacheFileEvict(t *testing.T) { } defer os.RemoveAll(tmpdir) e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}) + defer e.Close() workers := 8 epochs := 100 @@ -77,3 +82,90 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { e.VerifySeal(nil, head) } } + +func TestRemoteSealer(t *testing.T) { + ethash := NewTester() + defer ethash.Close() + api := &API{ethash} + if _, err := api.GetWork(); err != errNoMiningWork { + t.Error("expect to return an error indicate there is no mining work") + } + + head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(head) + + // Push new work. + ethash.Seal(nil, block, nil) + + var ( + work [3]string + err error + ) + if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { + t.Error("expect to return a mining work has same hash") + } + + if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { + t.Error("expect to return false when submit a fake solution") + } + + // Push new block with same block number to replace the original one. + head = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} + block = types.NewBlockWithHeader(head) + ethash.Seal(nil, block, nil) + + if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { + t.Error("expect to return the latest pushed work") + } + + // Push block with higher block number. + newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} + newBlock := types.NewBlockWithHeader(newHead) + ethash.Seal(nil, newBlock, nil) + + if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { + t.Error("expect to return false when submit a stale solution") + } +} + +func TestHashRate(t *testing.T) { + var ( + ethash = NewTester() + api = &API{ethash} + hashrate = []hexutil.Uint64{100, 200, 300} + expect uint64 + ids = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")} + ) + + defer ethash.Close() + + if tot := ethash.Hashrate(); tot != 0 { + t.Error("expect the result should be zero") + } + + for i := 0; i < len(hashrate); i += 1 { + if res := api.SubmitHashRate(hashrate[i], ids[i]); !res { + t.Error("remote miner submit hashrate failed") + } + expect += uint64(hashrate[i]) + } + if tot := ethash.Hashrate(); tot != float64(expect) { + t.Error("expect total hashrate should be same") + } +} + +func TestClosedRemoteSealer(t *testing.T) { + ethash := NewTester() + // Make sure exit channel has been listened + time.Sleep(1 * time.Second) + ethash.Close() + + api := &API{ethash} + if _, err := api.GetWork(); err != errEthashStopped { + t.Error("expect to return an error to indicate ethash is stopped") + } + + if res := api.SubmitHashRate(hexutil.Uint64(100), common.HexToHash("a")); res { + t.Error("expect to return false when submit hashrate to a stopped ethash") + } +} diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index b5e742d8bb..a9449d4060 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -18,11 +18,13 @@ package ethash import ( crand "crypto/rand" + "errors" "math" "math/big" "math/rand" "runtime" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -30,6 +32,11 @@ import ( "github.com/ethereum/go-ethereum/log" ) +var ( + errNoMiningWork = errors.New("no mining work available yet") + errInvalidSealResult = errors.New("invalid or stale proof-of-work solution") +) + // Seal implements consensus.Engine, attempting to find a nonce that satisfies // the block's difficulty requirements. func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) { @@ -45,7 +52,6 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop } // Create a runner and the multiple search threads it directs abort := make(chan struct{}) - found := make(chan *types.Block) ethash.lock.Lock() threads := ethash.threads @@ -64,12 +70,16 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop if threads < 0 { threads = 0 // Allows disabling local mining without extra logic around local/remote } + // Push new work to remote sealer + if ethash.workCh != nil { + ethash.workCh <- block + } var pend sync.WaitGroup for i := 0; i < threads; i++ { pend.Add(1) go func(id int, nonce uint64) { defer pend.Done() - ethash.mine(block, id, nonce, abort, found) + ethash.mine(block, id, nonce, abort, ethash.resultCh) }(i, uint64(ethash.rand.Int63())) } // Wait until sealing is terminated or a nonce is found @@ -78,7 +88,7 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop case <-stop: // Outside abort, stop all miner threads close(abort) - case result = <-found: + case result = <-ethash.resultCh: // One of the threads found a block, abort all others close(abort) case <-ethash.update: @@ -150,3 +160,136 @@ search: // during sealing so it's not unmapped while being read. runtime.KeepAlive(dataset) } + +// remote starts a standalone goroutine to handle remote mining related stuff. +func (ethash *Ethash) remote() { + var ( + works = make(map[common.Hash]*types.Block) + rates = make(map[common.Hash]hashrate) + currentWork *types.Block + ) + + // getWork returns a work package for external miner. + // + // The work package consists of 3 strings: + // result[0], 32 bytes hex encoded current block header pow-hash + // result[1], 32 bytes hex encoded seed hash used for DAG + // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty + getWork := func() ([3]string, error) { + var res [3]string + if currentWork == nil { + return res, errNoMiningWork + } + res[0] = currentWork.HashNoNonce().Hex() + res[1] = common.BytesToHash(SeedHash(currentWork.NumberU64())).Hex() + + // Calculate the "target" to be returned to the external sealer. + n := big.NewInt(1) + n.Lsh(n, 255) + n.Div(n, currentWork.Difficulty()) + n.Lsh(n, 1) + res[2] = common.BytesToHash(n.Bytes()).Hex() + + // Trace the seal work fetched by remote sealer. + works[currentWork.HashNoNonce()] = currentWork + return res, nil + } + + // submitWork verifies the submitted pow solution, returning + // whether the solution was accepted or not (not can be both a bad pow as well as + // any other error, like no pending work or stale mining result). + submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, hash common.Hash) bool { + // Make sure the work submitted is present + block := works[hash] + if block == nil { + log.Info("Work submitted but none pending", "hash", hash) + return false + } + + // Verify the correctness of submitted result. + header := block.Header() + header.Nonce = nonce + header.MixDigest = mixDigest + if err := ethash.VerifySeal(nil, header); err != nil { + log.Warn("Invalid proof-of-work submitted", "hash", hash, "err", err) + return false + } + + // Make sure the result channel is created. + if ethash.resultCh == nil { + log.Warn("Ethash result channel is empty, submitted mining result is rejected") + return false + } + + // Solutions seems to be valid, return to the miner and notify acceptance. + select { + case ethash.resultCh <- block.WithSeal(header): + delete(works, hash) + return true + default: + log.Info("Work submitted is stale", "hash", hash) + return false + } + } + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case block := <-ethash.workCh: + if currentWork != nil && block.ParentHash() != currentWork.ParentHash() { + // Start new round mining, throw out all previous work. + works = make(map[common.Hash]*types.Block) + } + // Update current work with new received block. + // Note same work can be past twice, happens when changing CPU threads. + currentWork = block + + case work := <-ethash.fetchWorkCh: + // Return current mining work to remote miner. + miningWork, err := getWork() + if err != nil { + work.errc <- err + } else { + work.res <- miningWork + } + + case result := <-ethash.submitWorkCh: + // Verify submitted PoW solution based on maintained mining blocks. + if submitWork(result.nonce, result.mixDigest, result.hash) { + result.errc <- nil + } else { + result.errc <- errInvalidSealResult + } + + case result := <-ethash.submitRateCh: + // Trace remote sealer's hash rate by submitted value. + rates[result.id] = hashrate{rate: result.rate, ping: time.Now()} + close(result.done) + + case req := <-ethash.fetchRateCh: + // Gather all hash rate submitted by remote sealer. + var total uint64 + for _, rate := range rates { + // this could overflow + total += rate.rate + } + req <- total + + case <-ticker.C: + // Clear stale submitted hash rate. + for id, rate := range rates { + if time.Since(rate.ping) > 10*time.Second { + delete(rates, id) + } + } + + case errc := <-ethash.exitCh: + // Exit remote loop if ethash is closed and return relevant error. + errc <- nil + log.Trace("Ethash remote sealer is exiting") + return + } + } +} diff --git a/core/vm/evm.go b/core/vm/evm.go index c29bb8886e..a2722537dd 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -184,7 +184,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas precompiles = PrecompiledContractsByzantium } if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 { - // Calling a non existing account, don't do antything, but ping the tracer + // Calling a non existing account, don't do anything, but ping the tracer if evm.vmConfig.Debug && evm.depth == 0 { evm.vmConfig.Tracer.CaptureStart(caller.Address(), addr, false, input, gas, value) evm.vmConfig.Tracer.CaptureEnd(ret, 0, 0, nil) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index ec5de94eda..48caadf1f1 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -214,6 +214,7 @@ func opBenchmark(bench *testing.B, op func(pc *uint64, interpreter *EVMInterpret ) env.interpreter = evmInterpreter + evmInterpreter.intPool = poolOfIntPools.get() // convert args byteArgs := make([][]byte, len(args)) for i, arg := range args { @@ -229,6 +230,7 @@ func opBenchmark(bench *testing.B, op func(pc *uint64, interpreter *EVMInterpret op(&pc, evmInterpreter, nil, nil, stack) stack.pop() } + poolOfIntPools.put(evmInterpreter.intPool) } func BenchmarkOpAdd64(b *testing.B) { @@ -474,6 +476,7 @@ func BenchmarkOpMstore(bench *testing.B) { ) env.interpreter = evmInterpreter + evmInterpreter.intPool = poolOfIntPools.get() mem.Resize(64) pc := uint64(0) memStart := big.NewInt(0) @@ -484,4 +487,5 @@ func BenchmarkOpMstore(bench *testing.B) { stack.pushN(value, memStart) opMstore(&pc, evmInterpreter, nil, mem, stack) } + poolOfIntPools.put(evmInterpreter.intPool) } diff --git a/eth/api.go b/eth/api.go index 0b6da456f6..c1fbcb6d40 100644 --- a/eth/api.go +++ b/eth/api.go @@ -34,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" @@ -70,16 +69,12 @@ func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { // PublicMinerAPI provides an API to control the miner. // It offers only methods that operate on data that pose no security risk when it is publicly accessible. type PublicMinerAPI struct { - e *Ethereum - agent *miner.RemoteAgent + e *Ethereum } // NewPublicMinerAPI create a new PublicMinerAPI instance. func NewPublicMinerAPI(e *Ethereum) *PublicMinerAPI { - agent := miner.NewRemoteAgent(e.BlockChain(), e.Engine()) - e.Miner().Register(agent) - - return &PublicMinerAPI{e, agent} + return &PublicMinerAPI{e} } // Mining returns an indication if this node is currently mining. @@ -87,37 +82,6 @@ func (api *PublicMinerAPI) Mining() bool { return api.e.IsMining() } -// SubmitWork can be used by external miner to submit their POW solution. It returns an indication if the work was -// accepted. Note, this is not an indication if the provided work was valid! -func (api *PublicMinerAPI) SubmitWork(nonce types.BlockNonce, solution, digest common.Hash) bool { - return api.agent.SubmitWork(nonce, digest, solution) -} - -// GetWork returns a work package for external miner. The work package consists of 3 strings -// result[0], 32 bytes hex encoded current block header pow-hash -// result[1], 32 bytes hex encoded seed hash used for DAG -// result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty -func (api *PublicMinerAPI) GetWork() ([3]string, error) { - if !api.e.IsMining() { - if err := api.e.StartMining(false); err != nil { - return [3]string{}, err - } - } - work, err := api.agent.GetWork() - if err != nil { - return work, fmt.Errorf("mining not ready: %v", err) - } - return work, nil -} - -// SubmitHashrate can be used for remote miners to submit their hash rate. This enables the node to report the combined -// hash rate of all miners which submit work through this node. It accepts the miner hash rate and an identifier which -// must be unique between nodes. -func (api *PublicMinerAPI) SubmitHashrate(hashrate hexutil.Uint64, id common.Hash) bool { - api.agent.SubmitHashrate(id, uint64(hashrate)) - return true -} - // PrivateMinerAPI provides private RPC methods to control the miner. // These methods can be abused by external users and must be considered insecure for use by untrusted users. type PrivateMinerAPI struct { @@ -132,7 +96,8 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { // Start the miner with the given number of threads. If threads is nil the number // of workers started is equal to the number of logical CPUs that are usable by // this process. If mining is already running, this method adjust the number of -// threads allowed to use. +// threads allowed to use and updates the minimum price required by the transaction +// pool. func (api *PrivateMinerAPI) Start(threads *int) error { // Set the number of threads if the seal engine supports it if threads == nil { @@ -153,7 +118,6 @@ func (api *PrivateMinerAPI) Start(threads *int) error { api.e.lock.RLock() price := api.e.gasPrice api.e.lock.RUnlock() - api.e.txPool.SetGasPrice(price) return api.e.StartMining(true) } @@ -198,7 +162,7 @@ func (api *PrivateMinerAPI) SetEtherbase(etherbase common.Address) bool { // GetHashrate returns the current hashrate of the miner. func (api *PrivateMinerAPI) GetHashrate() uint64 { - return uint64(api.e.miner.HashRate()) + return api.e.miner.HashRate() } // PrivateAdminAPI is the collection of Ethereum full node-related APIs diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 623e5ed1bd..722e2a6e32 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -297,7 +297,9 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl database.TrieDB().Reference(root, common.Hash{}) } // Dereference all past tries we ourselves are done working with - database.TrieDB().Dereference(proot) + if proot != (common.Hash{}) { + database.TrieDB().Dereference(proot) + } proot = root // TODO(karalabe): Do we need the preimages? Won't they accumulate too much? @@ -526,7 +528,9 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* return nil, err } database.TrieDB().Reference(root, common.Hash{}) - database.TrieDB().Dereference(proot) + if proot != (common.Hash{}) { + database.TrieDB().Dereference(proot) + } proot = root } nodes, imgs := database.TrieDB().Size() diff --git a/eth/backend.go b/eth/backend.go index a18abdfb56..32946a0ab3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -166,6 +166,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { return nil, err } + eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) eth.miner.SetExtra(makeExtraData(config.ExtraData)) @@ -411,6 +412,7 @@ func (s *Ethereum) Start(srvr *p2p.Server) error { func (s *Ethereum) Stop() error { s.bloomIndexer.Close() s.blockchain.Stop() + s.engine.Close() s.protocolManager.Stop() if s.lesServer != nil { s.lesServer.Stop() @@ -421,6 +423,5 @@ func (s *Ethereum) Stop() error { s.chainDb.Close() close(s.shutdownChan) - return nil } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 89ebceec7c..000e3728de 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -21,6 +21,7 @@ var Modules = map[string]string{ "admin": Admin_JS, "chequebook": Chequebook_JS, "clique": Clique_JS, + "ethash": Ethash_JS, "debug": Debug_JS, "eth": Eth_JS, "miner": Miner_JS, @@ -109,6 +110,34 @@ web3._extend({ }); ` +const Ethash_JS = ` +web3._extend({ + property: 'ethash', + methods: [ + new web3._extend.Method({ + name: 'getWork', + call: 'ethash_getWork', + params: 0 + }), + new web3._extend.Method({ + name: 'getHashrate', + call: 'ethash_getHashrate', + params: 0 + }), + new web3._extend.Method({ + name: 'submitWork', + call: 'ethash_submitWork', + params: 3, + }), + new web3._extend.Method({ + name: 'submitHashRate', + call: 'ethash_submitHashRate', + params: 2, + }), + ] +}); +` + const Admin_JS = ` web3._extend({ property: 'admin', @@ -123,6 +152,16 @@ web3._extend({ call: 'admin_removePeer', params: 1 }), + new web3._extend.Method({ + name: 'addTrustedPeer', + call: 'admin_addTrustedPeer', + params: 1 + }), + new web3._extend.Method({ + name: 'removeTrustedPeer', + call: 'admin_removeTrustedPeer', + params: 1 + }), new web3._extend.Method({ name: 'exportChain', call: 'admin_exportChain', diff --git a/les/backend.go b/les/backend.go index 35f67f29f8..952d92cc2a 100644 --- a/les/backend.go +++ b/les/backend.go @@ -248,6 +248,7 @@ func (s *LightEthereum) Stop() error { s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() + s.engine.Close() s.eventMux.Stop() diff --git a/miner/agent.go b/miner/agent.go index e3cebbd2e2..e922ea153c 100644 --- a/miner/agent.go +++ b/miner/agent.go @@ -18,7 +18,6 @@ package miner import ( "sync" - "sync/atomic" "github.com/ethereum/go-ethereum/consensus" @@ -28,32 +27,43 @@ import ( type CpuAgent struct { mu sync.Mutex - workCh chan *Work + taskCh chan *Package + returnCh chan<- *Package stop chan struct{} quitCurrentOp chan struct{} - returnCh chan<- *Result chain consensus.ChainReader engine consensus.Engine - isMining int32 // isMining indicates whether the agent is currently mining + started int32 // started indicates whether the agent is currently started } func NewCpuAgent(chain consensus.ChainReader, engine consensus.Engine) *CpuAgent { - miner := &CpuAgent{ + agent := &CpuAgent{ chain: chain, engine: engine, stop: make(chan struct{}, 1), - workCh: make(chan *Work, 1), + taskCh: make(chan *Package, 1), } - return miner + return agent } -func (self *CpuAgent) Work() chan<- *Work { return self.workCh } -func (self *CpuAgent) SetReturnCh(ch chan<- *Result) { self.returnCh = ch } +func (self *CpuAgent) AssignTask(p *Package) { + if atomic.LoadInt32(&self.started) == 1 { + self.taskCh <- p + } +} +func (self *CpuAgent) DeliverTo(ch chan<- *Package) { self.returnCh = ch } + +func (self *CpuAgent) Start() { + if !atomic.CompareAndSwapInt32(&self.started, 0, 1) { + return // agent already started + } + go self.update() +} func (self *CpuAgent) Stop() { - if !atomic.CompareAndSwapInt32(&self.isMining, 1, 0) { + if !atomic.CompareAndSwapInt32(&self.started, 1, 0) { return // agent already stopped } self.stop <- struct{}{} @@ -61,31 +71,24 @@ done: // Empty work channel for { select { - case <-self.workCh: + case <-self.taskCh: default: break done } } } -func (self *CpuAgent) Start() { - if !atomic.CompareAndSwapInt32(&self.isMining, 0, 1) { - return // agent already started - } - go self.update() -} - func (self *CpuAgent) update() { out: for { select { - case work := <-self.workCh: + case p := <-self.taskCh: self.mu.Lock() if self.quitCurrentOp != nil { close(self.quitCurrentOp) } self.quitCurrentOp = make(chan struct{}) - go self.mine(work, self.quitCurrentOp) + go self.mine(p, self.quitCurrentOp) self.mu.Unlock() case <-self.stop: self.mu.Lock() @@ -99,10 +102,11 @@ out: } } -func (self *CpuAgent) mine(work *Work, stop <-chan struct{}) { - if result, err := self.engine.Seal(self.chain, work.Block, stop); result != nil { - log.Info("Successfully sealed new block", "number", result.Number(), "hash", result.Hash()) - self.returnCh <- &Result{work, result} +func (self *CpuAgent) mine(p *Package, stop <-chan struct{}) { + var err error + if p.Block, err = self.engine.Seal(self.chain, p.Block, stop); p.Block != nil { + log.Info("Successfully sealed new block", "number", p.Block.Number(), "hash", p.Block.Hash()) + self.returnCh <- p } else { if err != nil { log.Warn("Block sealing failed", "err", err) @@ -110,10 +114,3 @@ func (self *CpuAgent) mine(work *Work, stop <-chan struct{}) { self.returnCh <- nil } } - -func (self *CpuAgent) GetHashRate() int64 { - if pow, ok := self.engine.(consensus.PoW); ok { - return int64(pow.Hashrate()) - } - return 0 -} diff --git a/miner/miner.go b/miner/miner.go index d9256e9787..4c5717c8ad 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -44,12 +44,9 @@ type Backend interface { // Miner creates blocks and searches for proof-of-work values. type Miner struct { - mux *event.TypeMux - - worker *worker - + mux *event.TypeMux + worker *worker coinbase common.Address - mining int32 eth Backend engine consensus.Engine @@ -62,7 +59,7 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con eth: eth, mux: mux, engine: engine, - worker: newWorker(config, engine, common.Address{}, eth, mux), + worker: newWorker(config, engine, eth, mux), canStart: 1, } miner.Register(NewCpuAgent(eth.BlockChain(), engine)) @@ -111,23 +108,16 @@ func (self *Miner) Start(coinbase common.Address) { log.Info("Network syncing, will start miner afterwards") return } - atomic.StoreInt32(&self.mining, 1) - - log.Info("Starting mining operation") self.worker.start() self.worker.commitNewWork() } func (self *Miner) Stop() { self.worker.stop() - atomic.StoreInt32(&self.mining, 0) atomic.StoreInt32(&self.shouldStart, 0) } func (self *Miner) Register(agent Agent) { - if self.Mining() { - agent.Start() - } self.worker.register(agent) } @@ -136,22 +126,14 @@ func (self *Miner) Unregister(agent Agent) { } func (self *Miner) Mining() bool { - return atomic.LoadInt32(&self.mining) > 0 + return self.worker.isRunning() } -func (self *Miner) HashRate() (tot int64) { +func (self *Miner) HashRate() uint64 { if pow, ok := self.engine.(consensus.PoW); ok { - tot += int64(pow.Hashrate()) + return uint64(pow.Hashrate()) } - // do we care this might race? is it worth we're rewriting some - // aspects of the worker/locking up agents so we can get an accurate - // hashrate? - for agent := range self.worker.agents { - if _, ok := agent.(*CpuAgent); !ok { - tot += agent.GetHashRate() - } - } - return + return 0 } func (self *Miner) SetExtra(extra []byte) error { diff --git a/miner/remote_agent.go b/miner/remote_agent.go deleted file mode 100644 index 287e7530c3..0000000000 --- a/miner/remote_agent.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package miner - -import ( - "errors" - "math/big" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" -) - -type hashrate struct { - ping time.Time - rate uint64 -} - -type RemoteAgent struct { - mu sync.Mutex - - quitCh chan struct{} - workCh chan *Work - returnCh chan<- *Result - - chain consensus.ChainReader - engine consensus.Engine - currentWork *Work - work map[common.Hash]*Work - - hashrateMu sync.RWMutex - hashrate map[common.Hash]hashrate - - running int32 // running indicates whether the agent is active. Call atomically -} - -func NewRemoteAgent(chain consensus.ChainReader, engine consensus.Engine) *RemoteAgent { - return &RemoteAgent{ - chain: chain, - engine: engine, - work: make(map[common.Hash]*Work), - hashrate: make(map[common.Hash]hashrate), - } -} - -func (a *RemoteAgent) SubmitHashrate(id common.Hash, rate uint64) { - a.hashrateMu.Lock() - defer a.hashrateMu.Unlock() - - a.hashrate[id] = hashrate{time.Now(), rate} -} - -func (a *RemoteAgent) Work() chan<- *Work { - return a.workCh -} - -func (a *RemoteAgent) SetReturnCh(returnCh chan<- *Result) { - a.returnCh = returnCh -} - -func (a *RemoteAgent) Start() { - if !atomic.CompareAndSwapInt32(&a.running, 0, 1) { - return - } - a.quitCh = make(chan struct{}) - a.workCh = make(chan *Work, 1) - go a.loop(a.workCh, a.quitCh) -} - -func (a *RemoteAgent) Stop() { - if !atomic.CompareAndSwapInt32(&a.running, 1, 0) { - return - } - close(a.quitCh) - close(a.workCh) -} - -// GetHashRate returns the accumulated hashrate of all identifier combined -func (a *RemoteAgent) GetHashRate() (tot int64) { - a.hashrateMu.RLock() - defer a.hashrateMu.RUnlock() - - // this could overflow - for _, hashrate := range a.hashrate { - tot += int64(hashrate.rate) - } - return -} - -func (a *RemoteAgent) GetWork() ([3]string, error) { - a.mu.Lock() - defer a.mu.Unlock() - - var res [3]string - - if a.currentWork != nil { - block := a.currentWork.Block - - res[0] = block.HashNoNonce().Hex() - seedHash := ethash.SeedHash(block.NumberU64()) - res[1] = common.BytesToHash(seedHash).Hex() - // Calculate the "target" to be returned to the external miner - n := big.NewInt(1) - n.Lsh(n, 255) - n.Div(n, block.Difficulty()) - n.Lsh(n, 1) - res[2] = common.BytesToHash(n.Bytes()).Hex() - - a.work[block.HashNoNonce()] = a.currentWork - return res, nil - } - return res, errors.New("No work available yet, don't panic.") -} - -// SubmitWork tries to inject a pow solution into the remote agent, returning -// whether the solution was accepted or not (not can be both a bad pow as well as -// any other error, like no work pending). -func (a *RemoteAgent) SubmitWork(nonce types.BlockNonce, mixDigest, hash common.Hash) bool { - a.mu.Lock() - defer a.mu.Unlock() - - // Make sure the work submitted is present - work := a.work[hash] - if work == nil { - log.Info("Work submitted but none pending", "hash", hash) - return false - } - // Make sure the Engine solutions is indeed valid - result := work.Block.Header() - result.Nonce = nonce - result.MixDigest = mixDigest - - if err := a.engine.VerifySeal(a.chain, result); err != nil { - log.Warn("Invalid proof-of-work submitted", "hash", hash, "err", err) - return false - } - block := work.Block.WithSeal(result) - - // Solutions seems to be valid, return to the miner and notify acceptance - a.returnCh <- &Result{work, block} - delete(a.work, hash) - - return true -} - -// loop monitors mining events on the work and quit channels, updating the internal -// state of the remote miner until a termination is requested. -// -// Note, the reason the work and quit channels are passed as parameters is because -// RemoteAgent.Start() constantly recreates these channels, so the loop code cannot -// assume data stability in these member fields. -func (a *RemoteAgent) loop(workCh chan *Work, quitCh chan struct{}) { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - for { - select { - case <-quitCh: - return - case work := <-workCh: - a.mu.Lock() - a.currentWork = work - a.mu.Unlock() - case <-ticker.C: - // cleanup - a.mu.Lock() - for hash, work := range a.work { - if time.Since(work.createdAt) > 7*(12*time.Second) { - delete(a.work, hash) - } - } - a.mu.Unlock() - - a.hashrateMu.Lock() - for id, hashrate := range a.hashrate { - if time.Since(hashrate.ping) > 10*time.Second { - delete(a.hashrate, id) - } - } - a.hashrateMu.Unlock() - } - } -} diff --git a/miner/worker.go b/miner/worker.go index 34329f8499..ae695f0198 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -51,18 +51,16 @@ const ( chainSideChanSize = 10 ) -// Agent can register themself with the worker +// Agent can register themselves with the worker type Agent interface { - Work() chan<- *Work - SetReturnCh(chan<- *Result) - Stop() + AssignTask(*Package) + DeliverTo(chan<- *Package) Start() - GetHashRate() int64 + Stop() } -// Work is the workers current environment and holds -// all of the current state information -type Work struct { +// Env is the workers current environment and holds all of the current state information. +type Env struct { config *params.ChainConfig signer types.Signer @@ -73,8 +71,6 @@ type Work struct { tcount int // tx count in cycle gasPool *core.GasPool // available gas used to pack transactions - Block *types.Block // the new block - header *types.Header txs []*types.Transaction receipts []*types.Receipt @@ -82,9 +78,11 @@ type Work struct { createdAt time.Time } -type Result struct { - Work *Work - Block *types.Block +// Package contains all information for consensus engine sealing and result submitting. +type Package struct { + Receipts []*types.Receipt + State *state.StateDB + Block *types.Block } // worker is the main object which takes care of applying messages to the new state @@ -102,10 +100,9 @@ type worker struct { chainHeadSub event.Subscription chainSideCh chan core.ChainSideEvent chainSideSub event.Subscription - wg sync.WaitGroup agents map[Agent]struct{} - recv chan *Result + recv chan *Package eth Backend chain *core.BlockChain @@ -116,7 +113,7 @@ type worker struct { extra []byte currentMu sync.Mutex - current *Work + current *Env snapshotMu sync.RWMutex snapshotBlock *types.Block @@ -128,11 +125,10 @@ type worker struct { unconfirmed *unconfirmedBlocks // set of locally mined blocks pending canonicalness confirmations // atomic status counters - mining int32 - atWork int32 + running int32 // The indicator whether the consensus engine is running or not. } -func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase common.Address, eth Backend, mux *event.TypeMux) *worker { +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { worker := &worker{ config: config, engine: engine, @@ -142,11 +138,10 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), chainDb: eth.ChainDb(), - recv: make(chan *Result, resultQueueSize), + recv: make(chan *Package, resultQueueSize), chain: eth.BlockChain(), proc: eth.BlockChain().Validator(), possibleUncles: make(map[common.Hash]*types.Block), - coinbase: coinbase, agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), } @@ -176,62 +171,50 @@ func (self *worker) setExtra(extra []byte) { } func (self *worker) pending() (*types.Block, *state.StateDB) { - if atomic.LoadInt32(&self.mining) == 0 { - // return a snapshot to avoid contention on currentMu mutex - self.snapshotMu.RLock() - defer self.snapshotMu.RUnlock() - return self.snapshotBlock, self.snapshotState.Copy() - } - - self.currentMu.Lock() - defer self.currentMu.Unlock() - return self.current.Block, self.current.state.Copy() + // return a snapshot to avoid contention on currentMu mutex + self.snapshotMu.RLock() + defer self.snapshotMu.RUnlock() + return self.snapshotBlock, self.snapshotState.Copy() } func (self *worker) pendingBlock() *types.Block { - if atomic.LoadInt32(&self.mining) == 0 { - // return a snapshot to avoid contention on currentMu mutex - self.snapshotMu.RLock() - defer self.snapshotMu.RUnlock() - return self.snapshotBlock - } - - self.currentMu.Lock() - defer self.currentMu.Unlock() - return self.current.Block + // return a snapshot to avoid contention on currentMu mutex + self.snapshotMu.RLock() + defer self.snapshotMu.RUnlock() + return self.snapshotBlock } func (self *worker) start() { self.mu.Lock() defer self.mu.Unlock() - - atomic.StoreInt32(&self.mining, 1) - - // spin up agents + atomic.StoreInt32(&self.running, 1) for agent := range self.agents { agent.Start() } } func (self *worker) stop() { - self.wg.Wait() - self.mu.Lock() defer self.mu.Unlock() - if atomic.LoadInt32(&self.mining) == 1 { - for agent := range self.agents { - agent.Stop() - } + + atomic.StoreInt32(&self.running, 0) + for agent := range self.agents { + agent.Stop() } - atomic.StoreInt32(&self.mining, 0) - atomic.StoreInt32(&self.atWork, 0) +} + +func (self *worker) isRunning() bool { + return atomic.LoadInt32(&self.running) == 1 } func (self *worker) register(agent Agent) { self.mu.Lock() defer self.mu.Unlock() self.agents[agent] = struct{}{} - agent.SetReturnCh(self.recv) + agent.DeliverTo(self.recv) + if self.isRunning() { + agent.Start() + } } func (self *worker) unregister(agent Agent) { @@ -266,7 +249,7 @@ func (self *worker) update() { // Note all transactions received may not be continuous with transactions // already included in the current mining block. These transactions will // be automatically eliminated. - if atomic.LoadInt32(&self.mining) == 0 { + if !self.isRunning() { self.currentMu.Lock() txs := make(map[common.Address]types.Transactions) for _, tx := range ev.Txs { @@ -298,26 +281,24 @@ func (self *worker) update() { func (self *worker) wait() { for { for result := range self.recv { - atomic.AddInt32(&self.atWork, -1) if result == nil { continue } block := result.Block - work := result.Work // Update the block hash in all logs since it is now available and not when the // receipt/log of individual transactions were created. - for _, r := range work.receipts { + for _, r := range result.Receipts { for _, l := range r.Logs { l.BlockHash = block.Hash() } } - for _, log := range work.state.Logs() { + for _, log := range result.State.Logs() { log.BlockHash = block.Hash() } self.currentMu.Lock() - stat, err := self.chain.WriteBlockWithState(block, work.receipts, work.state) + stat, err := self.chain.WriteBlockWithState(block, result.Receipts, result.State) self.currentMu.Unlock() if err != nil { log.Error("Failed writing block to chain", "err", err) @@ -327,7 +308,7 @@ func (self *worker) wait() { self.mux.Post(core.NewMinedBlockEvent{Block: block}) var ( events []interface{} - logs = work.state.Logs() + logs = result.State.Logs() ) events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) if stat == core.CanonStatTy { @@ -342,15 +323,9 @@ func (self *worker) wait() { } // push sends a new work task to currently live miner agents. -func (self *worker) push(work *Work) { - if atomic.LoadInt32(&self.mining) != 1 { - return - } +func (self *worker) push(p *Package) { for agent := range self.agents { - atomic.AddInt32(&self.atWork, 1) - if ch := agent.Work(); ch != nil { - ch <- work - } + agent.AssignTask(p) } } @@ -360,7 +335,7 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error if err != nil { return err } - work := &Work{ + env := &Env{ config: self.config, signer: types.NewEIP155Signer(self.config.ChainID), state: state, @@ -374,15 +349,15 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error // when 08 is processed ancestors contain 07 (quick block) for _, ancestor := range self.chain.GetBlocksFromHash(parent.Hash(), 7) { for _, uncle := range ancestor.Uncles() { - work.family.Add(uncle.Hash()) + env.family.Add(uncle.Hash()) } - work.family.Add(ancestor.Hash()) - work.ancestors.Add(ancestor.Hash()) + env.family.Add(ancestor.Hash()) + env.ancestors.Add(ancestor.Hash()) } // Keep track of transactions which return errors so they can be removed - work.tcount = 0 - self.current = work + env.tcount = 0 + self.current = env return nil } @@ -416,8 +391,12 @@ func (self *worker) commitNewWork() { Extra: self.extra, Time: big.NewInt(tstamp), } - // Only set the coinbase if we are mining (avoid spurious block rewards) - if atomic.LoadInt32(&self.mining) == 1 { + // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) + if self.isRunning() { + if self.coinbase == (common.Address{}) { + log.Error("Refusing to mine without etherbase") + return + } header.Coinbase = self.coinbase } if err := self.engine.Prepare(self.chain, header); err != nil { @@ -444,17 +423,10 @@ func (self *worker) commitNewWork() { return } // Create the current work task and check any fork transitions needed - work := self.current + env := self.current if self.config.DAOForkSupport && self.config.DAOForkBlock != nil && self.config.DAOForkBlock.Cmp(header.Number) == 0 { - misc.ApplyDAOHardFork(work.state) + misc.ApplyDAOHardFork(env.state) } - pending, err := self.eth.TxPool().Pending() - if err != nil { - log.Error("Failed to fetch pending transactions", "err", err) - return - } - txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) - work.commitTransactions(self.mux, txs, self.chain, self.coinbase) // compute uncles for the new block. var ( @@ -465,7 +437,7 @@ func (self *worker) commitNewWork() { if len(uncles) == 2 { break } - if err := self.commitUncle(work, uncle.Header()); err != nil { + if err := self.commitUncle(env, uncle.Header()); err != nil { log.Trace("Bad uncle found and will be removed", "hash", hash) log.Trace(fmt.Sprint(uncle)) @@ -478,32 +450,62 @@ func (self *worker) commitNewWork() { for _, hash := range badUncles { delete(self.possibleUncles, hash) } - // Create the new block to seal with the consensus engine - if work.Block, err = self.engine.Finalize(self.chain, header, work.state, work.txs, uncles, work.receipts); err != nil { + + var ( + emptyBlock *types.Block + fullBlock *types.Block + ) + + // Create an empty block based on temporary copied state for sealing in advance without waiting block + // execution finished. + emptyState := env.state.Copy() + if emptyBlock, err = self.engine.Finalize(self.chain, header, emptyState, nil, uncles, nil); err != nil { + log.Error("Failed to finalize block for temporary sealing", "err", err) + } else { + // Push empty work in advance without applying pending transaction. + // The reason is transactions execution can cost a lot and sealer need to + // take advantage of this part time. + if self.isRunning() { + log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) + self.push(&Package{nil, emptyState, emptyBlock}) + } + } + + // Fill the block with all available pending transactions. + pending, err := self.eth.TxPool().Pending() + if err != nil { + log.Error("Failed to fetch pending transactions", "err", err) + return + } + txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) + env.commitTransactions(self.mux, txs, self.chain, self.coinbase) + + // Create the full block to seal with the consensus engine + if fullBlock, err = self.engine.Finalize(self.chain, header, env.state, env.txs, uncles, env.receipts); err != nil { log.Error("Failed to finalize block for sealing", "err", err) return } // We only care about logging if we're actually mining. - if atomic.LoadInt32(&self.mining) == 1 { - log.Info("Commit new mining work", "number", work.Block.Number(), "txs", work.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) - self.unconfirmed.Shift(work.Block.NumberU64() - 1) + if self.isRunning() { + log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) + self.unconfirmed.Shift(fullBlock.NumberU64() - 1) + self.push(&Package{env.receipts, env.state, fullBlock}) } - self.push(work) self.updateSnapshot() } -func (self *worker) commitUncle(work *Work, uncle *types.Header) error { +func (self *worker) commitUncle(env *Env, uncle *types.Header) error { hash := uncle.Hash() - if work.uncles.Contains(hash) { + if env.uncles.Contains(hash) { return fmt.Errorf("uncle not unique") } - if !work.ancestors.Contains(uncle.ParentHash) { + if !env.ancestors.Contains(uncle.ParentHash) { return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) } - if work.family.Contains(hash) { + if env.family.Contains(hash) { return fmt.Errorf("uncle already in family (%x)", hash) } - work.uncles.Add(uncle.Hash()) + env.uncles.Add(uncle.Hash()) return nil } @@ -511,16 +513,25 @@ func (self *worker) updateSnapshot() { self.snapshotMu.Lock() defer self.snapshotMu.Unlock() + var uncles []*types.Header + self.current.uncles.Each(func(item interface{}) bool { + if header, ok := item.(*types.Header); ok { + uncles = append(uncles, header) + return true + } + return false + }) + self.snapshotBlock = types.NewBlock( self.current.header, self.current.txs, - nil, + uncles, self.current.receipts, ) self.snapshotState = self.current.state.Copy() } -func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { +func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { if env.gasPool == nil { env.gasPool = new(core.GasPool).AddGas(env.header.GasLimit) } @@ -605,7 +616,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB } } -func (env *Work) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { +func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { snap := env.state.Snapshot() receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) diff --git a/mobile/ethereum.go b/mobile/ethereum.go index 35a43d274d..59da852397 100644 --- a/mobile/ethereum.go +++ b/mobile/ethereum.go @@ -67,6 +67,7 @@ func (msg *CallMsg) SetData(data []byte) { msg.msg.Data = common.CopyBytes func (msg *CallMsg) SetTo(address *Address) { if address == nil { msg.msg.To = nil + return } msg.msg.To = &address.address } diff --git a/node/api.go b/node/api.go index f44c991539..2656c21203 100644 --- a/node/api.go +++ b/node/api.go @@ -59,7 +59,7 @@ func (api *PrivateAdminAPI) AddPeer(url string) (bool, error) { return true, nil } -// RemovePeer disconnects from a a remote node if the connection exists +// RemovePeer disconnects from a remote node if the connection exists func (api *PrivateAdminAPI) RemovePeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() @@ -75,6 +75,37 @@ func (api *PrivateAdminAPI) RemovePeer(url string) (bool, error) { return true, nil } +// AddTrustedPeer allows a remote node to always connect, even if slots are full +func (api *PrivateAdminAPI) AddTrustedPeer(url string) (bool, error) { + // Make sure the server is running, fail otherwise + server := api.node.Server() + if server == nil { + return false, ErrNodeStopped + } + node, err := discover.ParseNode(url) + if err != nil { + return false, fmt.Errorf("invalid enode: %v", err) + } + server.AddTrustedPeer(node) + return true, nil +} + +// RemoveTrustedPeer removes a remote node from the trusted peer set, but it +// does not disconnect it automatically. +func (api *PrivateAdminAPI) RemoveTrustedPeer(url string) (bool, error) { + // Make sure the server is running, fail otherwise + server := api.node.Server() + if server == nil { + return false, ErrNodeStopped + } + node, err := discover.ParseNode(url) + if err != nil { + return false, fmt.Errorf("invalid enode: %v", err) + } + server.RemoveTrustedPeer(node) + return true, nil +} + // PeerEvents creates an RPC subscription which receives peer events from the // node's p2p.Server func (api *PrivateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) { diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 8803daa56e..0a554bbeb4 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -160,7 +160,7 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) { // Find all non-empty buckets and get a fresh slice of their entries. var buckets [][]*Node - for _, b := range tab.buckets { + for _, b := range &tab.buckets { if len(b.entries) > 0 { buckets = append(buckets, b.entries[:]) } @@ -508,7 +508,7 @@ func (tab *Table) copyLiveNodes() { defer tab.mutex.Unlock() now := time.Now() - for _, b := range tab.buckets { + for _, b := range &tab.buckets { for _, n := range b.entries { if now.Sub(n.addedAt) >= seedMinTableTime { tab.db.updateNode(n) @@ -524,7 +524,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { // obviously correct. I believe that tree-based buckets would make // this easier to implement efficiently. close := &nodesByDistance{target: target} - for _, b := range tab.buckets { + for _, b := range &tab.buckets { for _, n := range b.entries { close.push(n, nresults) } @@ -533,7 +533,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { } func (tab *Table) len() (n int) { - for _, b := range tab.buckets { + for _, b := range &tab.buckets { n += len(b.entries) } return n diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index 9b0bd0c80a..4c39c05533 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -1228,7 +1228,7 @@ func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) { if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash { return nil, errors.New("topic hash mismatch") } - if data.Idx < 0 || int(data.Idx) >= len(data.Topics) { + if int(data.Idx) < 0 || int(data.Idx) >= len(data.Topics) { return nil, errors.New("topic index out of range") } return pongpkt.data.(*pong), nil diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index 001d193cc9..1a8137673d 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -355,7 +355,7 @@ func (tn *preminedTestnet) mine(target NodeID) { fmt.Printf(" target: %#v,\n", tn.target) fmt.Printf(" targetSha: %#v,\n", tn.targetSha) fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) - for ld, ns := range tn.dists { + for ld, ns := range &tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/table.go b/p2p/discv5/table.go index c8d234b936..c793be5082 100644 --- a/p2p/discv5/table.go +++ b/p2p/discv5/table.go @@ -81,7 +81,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash { if printTable { fmt.Println() } - for i, b := range tab.buckets { + for i, b := range &tab.buckets { entries += len(b.entries) if printTable { for _, e := range b.entries { @@ -93,7 +93,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash { prefix := binary.BigEndian.Uint64(tab.self.sha[0:8]) dist := ^uint64(0) entry := int(randUint(uint32(entries + 1))) - for _, b := range tab.buckets { + for _, b := range &tab.buckets { if entry < len(b.entries) { n := b.entries[entry] dist = binary.BigEndian.Uint64(n.sha[0:8]) ^ prefix @@ -121,7 +121,7 @@ func (tab *Table) readRandomNodes(buf []*Node) (n int) { // TODO: tree-based buckets would help here // Find all non-empty buckets and get a fresh slice of their entries. var buckets [][]*Node - for _, b := range tab.buckets { + for _, b := range &tab.buckets { if len(b.entries) > 0 { buckets = append(buckets, b.entries[:]) } @@ -175,7 +175,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { // obviously correct. I believe that tree-based buckets would make // this easier to implement efficiently. close := &nodesByDistance{target: target} - for _, b := range tab.buckets { + for _, b := range &tab.buckets { for _, n := range b.entries { close.push(n, nresults) } diff --git a/p2p/peer.go b/p2p/peer.go index e8d1f25404..482e3d5064 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -165,7 +165,7 @@ func (p *Peer) String() string { // Inbound returns true if the peer is an inbound connection func (p *Peer) Inbound() bool { - return p.rw.flags&inboundConn != 0 + return p.rw.is(inboundConn) } func newPeer(conn *conn, protocols []Protocol) *Peer { diff --git a/p2p/server.go b/p2p/server.go index 8f860d8f10..8f3a511f31 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -169,6 +170,8 @@ type Server struct { quit chan struct{} addstatic chan *discover.Node removestatic chan *discover.Node + addtrusted chan *discover.Node + removetrusted chan *discover.Node posthandshake chan *conn addpeer chan *conn delpeer chan peerDrop @@ -185,7 +188,7 @@ type peerDrop struct { requested bool // true if signaled by the peer } -type connFlag int +type connFlag int32 const ( dynDialedConn connFlag = 1 << iota @@ -250,7 +253,23 @@ func (f connFlag) String() string { } func (c *conn) is(f connFlag) bool { - return c.flags&f != 0 + flags := connFlag(atomic.LoadInt32((*int32)(&c.flags))) + return flags&f != 0 +} + +func (c *conn) set(f connFlag, val bool) { + for { + oldFlags := connFlag(atomic.LoadInt32((*int32)(&c.flags))) + flags := oldFlags + if val { + flags |= f + } else { + flags &= ^f + } + if atomic.CompareAndSwapInt32((*int32)(&c.flags), int32(oldFlags), int32(flags)) { + return + } + } } // Peers returns all connected peers. @@ -300,6 +319,23 @@ func (srv *Server) RemovePeer(node *discover.Node) { } } +// AddTrustedPeer adds the given node to a reserved whitelist which allows the +// node to always connect, even if the slot are full. +func (srv *Server) AddTrustedPeer(node *discover.Node) { + select { + case srv.addtrusted <- node: + case <-srv.quit: + } +} + +// RemoveTrustedPeer removes the given node from the trusted peer set. +func (srv *Server) RemoveTrustedPeer(node *discover.Node) { + select { + case srv.removetrusted <- node: + case <-srv.quit: + } +} + // SubscribePeers subscribes the given channel to peer events func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription { return srv.peerFeed.Subscribe(ch) @@ -411,6 +447,8 @@ func (srv *Server) Start() (err error) { srv.posthandshake = make(chan *conn) srv.addstatic = make(chan *discover.Node) srv.removestatic = make(chan *discover.Node) + srv.addtrusted = make(chan *discover.Node) + srv.removetrusted = make(chan *discover.Node) srv.peerOp = make(chan peerOpFunc) srv.peerOpDone = make(chan struct{}) @@ -547,8 +585,7 @@ func (srv *Server) run(dialstate dialer) { queuedTasks []task // tasks that can't run yet ) // Put trusted nodes into a map to speed up checks. - // Trusted peers are loaded on startup and cannot be - // modified while the server is running. + // Trusted peers are loaded on startup or added via AddTrustedPeer RPC. for _, n := range srv.TrustedNodes { trusted[n.ID] = true } @@ -600,12 +637,32 @@ running: case n := <-srv.removestatic: // This channel is used by RemovePeer to send a // disconnect request to a peer and begin the - // stop keeping the node connected + // stop keeping the node connected. srv.log.Trace("Removing static node", "node", n) dialstate.removeStatic(n) if p, ok := peers[n.ID]; ok { p.Disconnect(DiscRequested) } + case n := <-srv.addtrusted: + // This channel is used by AddTrustedPeer to add an enode + // to the trusted node set. + srv.log.Trace("Adding trusted node", "node", n) + trusted[n.ID] = true + // Mark any already-connected peer as trusted + if p, ok := peers[n.ID]; ok { + p.rw.set(trustedConn, true) + } + case n := <-srv.removetrusted: + // This channel is used by RemoveTrustedPeer to remove an enode + // from the trusted node set. + srv.log.Trace("Removing trusted node", "node", n) + if _, ok := trusted[n.ID]; ok { + delete(trusted, n.ID) + } + // Unmark any already-connected peer as trusted + if p, ok := peers[n.ID]; ok { + p.rw.set(trustedConn, false) + } case op := <-srv.peerOp: // This channel is used by Peers and PeerCount. op(peers) diff --git a/p2p/server_test.go b/p2p/server_test.go index 10c36528eb..3f24a79bae 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -148,7 +148,8 @@ func TestServerDial(t *testing.T) { // tell the server to connect tcpAddr := listener.Addr().(*net.TCPAddr) - srv.AddPeer(&discover.Node{ID: remid, IP: tcpAddr.IP, TCP: uint16(tcpAddr.Port)}) + node := &discover.Node{ID: remid, IP: tcpAddr.IP, TCP: uint16(tcpAddr.Port)} + srv.AddPeer(node) select { case conn := <-accepted: @@ -170,6 +171,29 @@ func TestServerDial(t *testing.T) { if !reflect.DeepEqual(peers, []*Peer{peer}) { t.Errorf("Peers mismatch: got %v, want %v", peers, []*Peer{peer}) } + + // Test AddTrustedPeer/RemoveTrustedPeer and changing Trusted flags + // Particularly for race conditions on changing the flag state. + if peer := srv.Peers()[0]; peer.Info().Network.Trusted { + t.Errorf("peer is trusted prematurely: %v", peer) + } + done := make(chan bool) + go func() { + srv.AddTrustedPeer(node) + if peer := srv.Peers()[0]; !peer.Info().Network.Trusted { + t.Errorf("peer is not trusted after AddTrustedPeer: %v", peer) + } + srv.RemoveTrustedPeer(node) + if peer := srv.Peers()[0]; peer.Info().Network.Trusted { + t.Errorf("peer is trusted after RemoveTrustedPeer: %v", peer) + } + done <- true + }() + // Trigger potential race conditions + peer = srv.Peers()[0] + _ = peer.Inbound() + _ = peer.Info() + <-done case <-time.After(1 * time.Second): t.Error("server did not launch peer within one second") } @@ -351,7 +375,8 @@ func TestServerAtCap(t *testing.T) { } } // Try inserting a non-trusted connection. - c := newconn(randomID()) + anotherID := randomID() + c := newconn(anotherID) if err := srv.checkpoint(c, srv.posthandshake); err != DiscTooManyPeers { t.Error("wrong error for insert:", err) } @@ -364,6 +389,87 @@ func TestServerAtCap(t *testing.T) { t.Error("Server did not set trusted flag") } + // Remove from trusted set and try again + srv.RemoveTrustedPeer(&discover.Node{ID: trustedID}) + c = newconn(trustedID) + if err := srv.checkpoint(c, srv.posthandshake); err != DiscTooManyPeers { + t.Error("wrong error for insert:", err) + } + + // Add anotherID to trusted set and try again + srv.AddTrustedPeer(&discover.Node{ID: anotherID}) + c = newconn(anotherID) + if err := srv.checkpoint(c, srv.posthandshake); err != nil { + t.Error("unexpected error for trusted conn @posthandshake:", err) + } + if !c.is(trustedConn) { + t.Error("Server did not set trusted flag") + } +} + +func TestServerPeerLimits(t *testing.T) { + srvkey := newkey() + + clientid := randomID() + clientnode := &discover.Node{ID: clientid} + + var tp *setupTransport = &setupTransport{ + id: clientid, + phs: &protoHandshake{ + ID: clientid, + // Force "DiscUselessPeer" due to unmatching caps + // Caps: []Cap{discard.cap()}, + }, + } + var flags connFlag = dynDialedConn + var dialDest *discover.Node = &discover.Node{ID: clientid} + + srv := &Server{ + Config: Config{ + PrivateKey: srvkey, + MaxPeers: 0, + NoDial: true, + Protocols: []Protocol{discard}, + }, + newTransport: func(fd net.Conn) transport { return tp }, + log: log.New(), + } + if err := srv.Start(); err != nil { + t.Fatalf("couldn't start server: %v", err) + } + defer srv.Stop() + + // Check that server is full (MaxPeers=0) + conn, _ := net.Pipe() + srv.SetupConn(conn, flags, dialDest) + if tp.closeErr != DiscTooManyPeers { + t.Errorf("unexpected close error: %q", tp.closeErr) + } + conn.Close() + + srv.AddTrustedPeer(clientnode) + + // Check that server allows a trusted peer despite being full. + conn, _ = net.Pipe() + srv.SetupConn(conn, flags, dialDest) + if tp.closeErr == DiscTooManyPeers { + t.Errorf("failed to bypass MaxPeers with trusted node: %q", tp.closeErr) + } + + if tp.closeErr != DiscUselessPeer { + t.Errorf("unexpected close error: %q", tp.closeErr) + } + conn.Close() + + srv.RemoveTrustedPeer(clientnode) + + // Check that server is full again. + conn, _ = net.Pipe() + srv.SetupConn(conn, flags, dialDest) + if tp.closeErr != DiscTooManyPeers { + t.Errorf("unexpected close error: %q", tp.closeErr) + } + conn.Close() } func TestServerSetupConn(t *testing.T) { diff --git a/params/version.go b/params/version.go index ea838d89f8..44c148e9c3 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 13 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 14 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go index 8276a9eadd..9c21c12d5d 100644 --- a/rpc/client_example_test.go +++ b/rpc/client_example_test.go @@ -66,7 +66,7 @@ func subscribeBlocks(client *rpc.Client, subch chan Block) { defer cancel() // Subscribe to new blocks. - sub, err := client.EthSubscribe(ctx, subch, "newBlocks") + sub, err := client.EthSubscribe(ctx, subch, "newHeads") if err != nil { fmt.Println("subscribe error:", err) return diff --git a/swarm/README.md b/swarm/README.md index f2d1890434..e81963217f 100644 --- a/swarm/README.md +++ b/swarm/README.md @@ -7,6 +7,21 @@ Swarm is a distributed storage platform and content distribution service, a nati [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethersphere/orange-lounge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +## Table of Contents + +* [Building the source](#building-the-source) +* [Running Swarm](#running-swarm) +* [Documentation](#documentation) +* [Developers Guide](#developers-guide) + * [Go Environment](#development-environment) + * [Vendored Dependencies](#vendored-dependencies) + * [Testing](#testing) + * [Profiling Swarm](#profiling-swarm) + * [Metrics and Instrumentation in Swarm](#metrics-and-instrumentation-in-swarm) +* [Public Gateways](#public-gateways) +* [Swarm Dapps](#swarm-dapps) +* [Contributing](#contributing) +* [License](#license) ## Building the source @@ -16,13 +31,187 @@ Building Swarm requires Go (version 1.10 or later). go install github.com/ethereum/go-ethereum/cmd/swarm +## Running Swarm + +Going through all the possible command line flags is out of scope here, but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own Swarm node. + +To run Swarm you need an Ethereum account. You can create a new account by running the following command: + + geth account new + +You will be prompted for a password: + + Your new account is locked with a password. Please give a password. Do not forget this password. + Passphrase: + Repeat passphrase: + +Once you have specified the password, the output will be the Ethereum address representing that account. For example: + + Address: {2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1} + +Using this account, connect to Swarm with + + swarm --bzzaccount + + # in our example + + swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 + + +### Verifying that your local Swarm node is running + +When running, Swarm is accessible through an HTTP API on port 8500. + +Confirm that it is up and running by pointing your browser to http://localhost:8500 + +### Ethereum Name Service resolution + +The Ethereum Name Service is the Ethereum equivalent of DNS in the classic web. In order to use ENS to resolve names to Swarm content hashes (e.g. `bzz://theswarm.eth`), `swarm` has to connect to a `geth` instance, which is synced with the Ethereum mainnet. This is done using the `--ens-api` flag. + + swarm --bzzaccount \ + --ens-api '$HOME/.ethereum/geth.ipc' + + # in our example + + swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 \ + --ens-api '$HOME/.ethereum/geth.ipc' + +For more information on usage, features or command line flags, please consult the Documentation. + ## Documentation Swarm documentation can be found at [https://swarm-guide.readthedocs.io](https://swarm-guide.readthedocs.io). -## Contribution +## Developers Guide + +### Go Environment + +We assume that you have Go v1.10 installed, and `GOPATH` is set. + +You must have your working copy under `$GOPATH/src/github.com/ethereum/go-ethereum`. + +Most likely you will be working from your fork of `go-ethereum`, let's say from `github.com/nirname/go-ethereum`. Clone or move your fork into the right place: + +``` +git clone git@github.com:nirname/go-ethereum.git $GOPATH/src/github.com/ethereum/go-ethereum +``` + + +### Vendored Dependencies + +All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them. + +If you want to add a new dependency, run `govendor fetch `, then commit the result. + +If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`. + + +### Testing + +This section explains how to run unit, integration, and end-to-end tests in your development sandbox. + +Testing one library: + +``` +go test -v -cpu 4 ./swarm/api +``` + +Note: Using options -cpu (number of cores allowed) and -v (logging even if no error) is recommended. + +Testing only some methods: + +``` +go test -v -cpu 4 ./eth -run TestMethod +``` + +Note: here all tests with prefix TestMethod will be run, so if you got TestMethod, TestMethod1, then both! + +Running benchmarks: + +``` +go test -v -cpu 4 -bench . -run BenchmarkJoin +``` + + +### Profiling Swarm + +This section explains how to add Go `pprof` profiler to Swarm + +If `swarm` is started with the `--pprof` option, a debugging HTTP server is made available on port 6060. + +You can bring up http://localhost:6060/debug/pprof to see the heap, running routines etc. + +By clicking full goroutine stack dump (clicking http://localhost:6060/debug/pprof/goroutine?debug=2) you can generate trace that is useful for debugging. + + +### Metrics and Instrumentation in Swarm + +This section explains how to visualize and use existing Swarm metrics and how to instrument Swarm with a new metric. + +Swarm metrics system is based on the `go-metrics` library. + +The most common types of measurements we use in Swarm are `counters` and `resetting timers`. Consult the `go-metrics` documentation for full reference of available types. + +``` +# incrementing a counter +metrics.GetOrRegisterCounter("network.stream.received_chunks", nil).Inc(1) + +# measuring latency with a resetting timer +start := time.Now() +t := metrics.GetOrRegisterResettingTimer("http.request.GET.time"), nil) +... +t := UpdateSince(start) +``` + +#### Visualizing metrics + +Swarm supports an InfluxDB exporter. Consult the help section to learn about the command line arguments used to configure it: + +``` +swarm --help | grep metrics +``` + +We use Grafana and InfluxDB to visualise metrics reported by Swarm. We keep our Grafana dashboards under version control at `./swarm/grafana_dashboards`. You could use them or design your own. + +We have built a tool to help with automatic start of Grafana and InfluxDB and provisioning of dashboards at https://github.com/nonsense/stateth , which requires that you have Docker installed. + +Once you have `stateth` installed, and you have Docker running locally, you have to: + +1. Run `stateth` and keep it running in the background +``` +stateth --rm --grafana-dashboards-folder $GOPATH/src/github.com/ethereum/go-ethereum/swarm/grafana_dashboards --influxdb-database metrics +``` + +2. Run `swarm` with at least the following params: +``` +--metrics \ +--metrics.influxdb.export \ +--metrics.influxdb.endpoint "http://localhost:8086" \ +--metrics.influxdb.username "admin" \ +--metrics.influxdb.password "admin" \ +--metrics.influxdb.database "metrics" +``` + +3. Open Grafana at http://localhost:3000 and view the dashboards to gain insight into Swarm. + + +## Public Gateways + +Swarm offers a local HTTP proxy API that Dapps can use to interact with Swarm. The Ethereum Foundation is hosting a public gateway, which allows free access so that people can try Swarm without running their own node. + +The Swarm public gateways are temporary and users should not rely on their existence for production services. + +The Swarm public gateway can be found at https://swarm-gateways.net and is always running the latest `stable` Swarm release. + +## Swarm Dapps + +You can find a few reference Swarm decentralised applications at: https://swarm-gateways.net/bzz:/swarmapps.eth + +Their source code can be found at: https://github.com/ethersphere/swarm-dapps + +## Contributing Thank you for considering to help out with the source code! We welcome contributions from anyone on the internet, and are grateful for even the smallest of fixes! diff --git a/swarm/api/api.go b/swarm/api/api.go index 05a38b5e1e..99d971b105 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -339,8 +339,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) - return + return nil, "", http.StatusNotFound, nil, err } log.Debug("trie getting entry", "key", manifestAddr, "path", path) @@ -526,6 +525,10 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err return nil }) + // close tar writer before closing pipew + // to flush remaining data to pipew + // regardless of error value + tw.Close() if err != nil { apiGetTarFail.Inc(1) pipew.CloseWithError(err) @@ -701,11 +704,12 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] return fkey, newMkey.String(), nil } -func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) { +func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath, defaultPath string, mw *ManifestWriter) (storage.Address, error) { apiUploadTarCount.Inc(1) var contentKey storage.Address tr := tar.NewReader(bodyReader) defer bodyReader.Close() + var defaultPathFound bool for { hdr, err := tr.Next() if err == io.EOF { @@ -734,6 +738,25 @@ func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestP apiUploadTarFail.Inc(1) return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err) } + if hdr.Name == defaultPath { + entry := &ManifestEntry{ + Hash: contentKey.Hex(), + Path: "", // default entry + ContentType: hdr.Xattrs["user.swarm.content-type"], + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: hdr.ModTime, + } + contentKey, err = mw.AddEntry(ctx, nil, entry) + if err != nil { + apiUploadTarFail.Inc(1) + return nil, fmt.Errorf("error adding default manifest entry from tar stream: %s", err) + } + defaultPathFound = true + } + } + if defaultPath != "" && !defaultPathFound { + return contentKey, fmt.Errorf("default path %q not found", defaultPath) } return contentKey, nil } diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index b3a5e929d0..8a9efe3608 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -138,7 +138,7 @@ func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, er if file.Size <= 0 { return "", errors.New("file size must be greater than zero") } - return c.TarUpload(manifest, &FileUploader{file}, toEncrypt) + return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt) } // Download downloads a file with the given path from the swarm manifest with @@ -175,7 +175,15 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo } else if !stat.IsDir() { return "", fmt.Errorf("not a directory: %s", dir) } - return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt) + if defaultPath != "" { + if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir) + } + return "", fmt.Errorf("default path: %v", err) + } + } + return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt) } // DownloadDirectory downloads the files contained in a swarm manifest under @@ -389,21 +397,11 @@ func (u UploaderFunc) Upload(upload UploadFn) error { // DirectoryUploader uploads all files in a directory, optionally uploading // a file to the default path type DirectoryUploader struct { - Dir string - DefaultPath string + Dir string } // Upload performs the upload of the directory and default path func (d *DirectoryUploader) Upload(upload UploadFn) error { - if d.DefaultPath != "" { - file, err := Open(d.DefaultPath) - if err != nil { - return err - } - if err := upload(file); err != nil { - return err - } - } return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { if err != nil { return err @@ -441,7 +439,7 @@ type UploadFn func(file *File) error // TarUpload uses the given Uploader to upload files to swarm as a tar stream, // returning the resulting manifest hash -func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) { +func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { reqR, reqW := io.Pipe() defer reqR.Close() addr := hash @@ -458,6 +456,11 @@ func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (stri return "", err } req.Header.Set("Content-Type", "application/x-tar") + if defaultPath != "" { + q := req.URL.Query() + q.Set("defaultpath", defaultPath) + req.URL.RawQuery = q.Encode() + } // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index dc608e3f1f..ae82a91d79 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -194,7 +194,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { // upload the directory client := NewClient(srv.URL) - defaultPath := filepath.Join(dir, testDirFiles[0]) + defaultPath := testDirFiles[0] hash, err := client.UploadDirectory(dir, defaultPath, "", false) if err != nil { t.Fatalf("error uploading directory: %s", err) diff --git a/swarm/api/config.go b/swarm/api/config.go index 939285e09c..bdfffdd05f 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -63,6 +63,7 @@ type Config struct { SwapEnabled bool SyncEnabled bool DeliverySkipCheck bool + LightNodeEnabled bool SyncUpdateDelay time.Duration SwapAPI string Cors string diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go deleted file mode 100644 index 254a0e8d48..0000000000 --- a/swarm/api/http/error.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -/* -Show nicely (but simple) formatted HTML error pages (or respond with JSON -if the appropriate `Accept` header is set)) for the http package. -*/ -package http - -import ( - "encoding/json" - "fmt" - "html/template" - "net/http" - "strings" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/api" - l "github.com/ethereum/go-ethereum/swarm/log" -) - -//templateMap holds a mapping of an HTTP error code to a template -var templateMap map[int]*template.Template -var caseErrors []CaseError - -//metrics variables -var ( - htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) - jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) -) - -//parameters needed for formatting the correct HTML page -type ResponseParams struct { - Msg string - Code int - Timestamp string - template *template.Template - Details template.HTML -} - -//a custom error case struct that would be used to store validators and -//additional error info to display with client responses. -type CaseError struct { - Validator func(*Request) bool - Msg func(*Request) string -} - -//we init the error handling right on boot time, so lookup and http response is fast -func init() { - initErrHandling() -} - -func initErrHandling() { - //pages are saved as strings - get these strings - genErrPage := GetGenericErrorPage() - notFoundPage := GetNotFoundErrorPage() - multipleChoicesPage := GetMultipleChoicesErrorPage() - //map the codes to the available pages - tnames := map[int]string{ - 0: genErrPage, //default - http.StatusBadRequest: genErrPage, - http.StatusNotFound: notFoundPage, - http.StatusMultipleChoices: multipleChoicesPage, - http.StatusInternalServerError: genErrPage, - } - templateMap = make(map[int]*template.Template) - for code, tname := range tnames { - //assign formatted HTML to the code - templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname)) - } - - caseErrors = []CaseError{ - { - Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") }, - Msg: func(r *Request) string { - uriCopy := r.uri - uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x") - return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.
- Please click here if your browser does not redirect you.`, "/"+uriCopy.String()) - }, - }} -} - -//ValidateCaseErrors is a method that process the request object through certain validators -//that assert if certain conditions are met for further information to log as an error -func ValidateCaseErrors(r *Request) string { - for _, err := range caseErrors { - if err.Validator(r) { - return err.Msg(r) - } - } - - return "" -} - -//ShowMultipeChoices is used when a user requests a resource in a manifest which results -//in ambiguous results. It returns a HTML page with clickable links of each of the entry -//in the manifest which fits the request URI ambiguity. -//For example, if the user requests bzz://read and that manifest contains entries -//"readme.md" and "readinglist.txt", a HTML page is returned with this two links. -//This only applies if the manifest has no default entry -func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) { - msg := "" - if list.Entries == nil { - Respond(w, req, "Could not resolve", http.StatusInternalServerError) - return - } - //make links relative - //requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt" - //to get clickable links, need to remove the ambiguous path, i.e. "read" - idx := strings.LastIndex(req.RequestURI, "/") - if idx == -1 { - Respond(w, req, "Internal Server Error", http.StatusInternalServerError) - return - } - //remove ambiguous part - base := req.RequestURI[:idx+1] - for _, e := range list.Entries { - //create clickable link for each entry - msg += "" + e.Path + "
" - } - Respond(w, req, msg, http.StatusMultipleChoices) -} - -//Respond is used to show an HTML page to a client. -//If there is an `Accept` header of `application/json`, JSON will be returned instead -//The function just takes a string message which will be displayed in the error page. -//The code is used to evaluate which template will be displayed -//(and return the correct HTTP status code) -func Respond(w http.ResponseWriter, req *Request, msg string, code int) { - additionalMessage := ValidateCaseErrors(req) - switch code { - case http.StatusInternalServerError: - log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code) - case http.StatusMultipleChoices: - log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code) - listURI := api.URI{ - Scheme: "bzz-list", - Addr: req.uri.Addr, - Path: req.uri.Path, - } - additionalMessage = fmt.Sprintf(`multiple choices`, listURI.String()) - default: - log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code) - } - - if code >= 400 { - w.Header().Del("Cache-Control") //avoid sending cache headers for errors! - w.Header().Del("ETag") - } - - respond(w, &req.Request, &ResponseParams{ - Code: code, - Msg: msg, - Details: template.HTML(additionalMessage), - Timestamp: time.Now().Format(time.RFC1123), - template: getTemplate(code), - }) -} - -//evaluate if client accepts html or json response -func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { - w.WriteHeader(params.Code) - if r.Header.Get("Accept") == "application/json" { - respondJSON(w, params) - } else { - respondHTML(w, params) - } -} - -//return a HTML page -func respondHTML(w http.ResponseWriter, params *ResponseParams) { - htmlCounter.Inc(1) - err := params.template.Execute(w, params) - if err != nil { - log.Error(err.Error()) - } -} - -//return JSON -func respondJSON(w http.ResponseWriter, params *ResponseParams) { - jsonCounter.Inc(1) - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(params) -} - -//get the HTML template for a given code -func getTemplate(code int) *template.Template { - if val, tmpl := templateMap[code]; tmpl { - return val - } - return templateMap[0] -} diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go deleted file mode 100644 index 78f24065a9..0000000000 --- a/swarm/api/http/error_templates.go +++ /dev/null @@ -1,559 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -/* -We use html templates to handle simple but as informative as possible error pages. - -To eliminate circular dependency in case of an error, we don't store error pages on swarm. -We can't save the error pages as html files on disk, or when deploying compiled binaries -they won't be found. - -For this reason we resort to save the HTML error pages as strings, which then can be -parsed by Go's html/template package -*/ -package http - -//This returns the HTML for generic errors -func GetGenericErrorPage() string { - page := ` - - - - - - - - - - - Swarm::HTTP Error Page - - - - -
- -
-
- -
-
-

There was a problem serving the requested page

-
-
-
{{.Timestamp}}
-
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - -
- Hmmmmm....Swarm was not able to serve your request! -
- Error message: -
- {{.Msg}} -
- {{.Details}} -
- Error code: -
- {{.Code}} -
-
-
- -
-

- Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
- Swarm -

-
- - -
- - - -` - return page -} - -//This returns the HTML for a 404 Not Found error -func GetNotFoundErrorPage() string { - page := ` - - - - - - - - - - - Swarm::404 HTTP Not Found - - - - - - -
-
- -
-
-

Resource Not Found

-
-
-
{{.Timestamp}}
-
-
- - -
- - - - - - - - - - - - - - - - - - - - - -
- Unfortunately, the resource you were trying to access could not be found on swarm. -
- {{.Msg}} -
- {{.Details}} -
- Error code: -
- {{.Code}} -
-
-
-
- -
-

- Swarm: Serverless Hosting Incentivised peer-to-peer Storage and Content Distribution -

-
- - - - -` - return page -} - -//This returns the HTML for a page listing disambiguation options -//i.e. if user requested bzz://read and the manifest contains "readme.md" and "readinglist.txt", -//this page is returned with a clickable list the existing disambiguation links in the manifest -func GetMultipleChoicesErrorPage() string { - page := ` - - - - - - - - - - - Swarm::HTTP Disambiguation Page - - - - - - -
-
- -
-
-

Swarm: disambiguation

-
-
-
{{.Timestamp}}
-
-
- - -
- - - - - - - - - - - - - -
- Your request may refer to {{ .Details}}. -
- Error code: -
- {{.Code}} -
-
-
-
- -
-

- Swarm: Serverless Hosting Incentivised peer-to-peer Storage and Content Distribution -

-
- - - - -` - return page -} diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go new file mode 100644 index 0000000000..c0d8d1a408 --- /dev/null +++ b/swarm/api/http/middleware.go @@ -0,0 +1,95 @@ +package http + +import ( + "fmt" + "net/http" + "runtime/debug" + "strings" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/spancontext" + "github.com/pborman/uuid" +) + +// Adapt chains h (main request handler) main handler to adapters (middleware handlers) +// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first) +func Adapt(h http.Handler, adapters ...Adapter) http.Handler { + for i := range adapters { + adapter := adapters[len(adapters)-1-i] + h = adapter(h) + } + return h +} + +type Adapter func(http.Handler) http.Handler + +func SetRequestID(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8])) + metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1) + log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI) + + h.ServeHTTP(w, r) + }) +} + +func ParseURI(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) + return + } + if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") { + uri.Addr = strings.TrimPrefix(uri.Addr, "0x") + + msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.
+ Please click here if your browser does not redirect you within 5 seconds.`, "/"+uri.String()) + w.WriteHeader(http.StatusNotFound) + w.Write([]byte(msg)) + return + } + + ctx := r.Context() + r = r.WithContext(SetURI(ctx, uri)) + log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme) + + h.ServeHTTP(w, r) + }) +} + +func InitLoggingResponseWriter(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writer := newLoggingResponseWriter(w) + h.ServeHTTP(writer, r) + log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + }) +} + +func InstrumentOpenTracing(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + uri := GetURI(r.Context()) + if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") { + h.ServeHTTP(w, r) // soft fail + return + } + spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme) + ctx, sp := spancontext.StartSpan(r.Context(), spanName) + defer sp.Finish() + h.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func RecoverPanic(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header) + } + }() + h.ServeHTTP(w, r) + }) +} diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go new file mode 100644 index 0000000000..f050e706a8 --- /dev/null +++ b/swarm/api/http/response.go @@ -0,0 +1,133 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package http + +import ( + "encoding/json" + "fmt" + "html/template" + "net/http" + "strings" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api" +) + +var ( + htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) + jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) + plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil) +) + +type ResponseParams struct { + Msg template.HTML + Code int + Timestamp string + template *template.Template + Details template.HTML +} + +// ShowMultipleChoices is used when a user requests a resource in a manifest which results +// in ambiguous results. It returns a HTML page with clickable links of each of the entry +// in the manifest which fits the request URI ambiguity. +// For example, if the user requests bzz://read and that manifest contains entries +// "readme.md" and "readinglist.txt", a HTML page is returned with this two links. +// This only applies if the manifest has no default entry +func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) { + log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) + msg := "" + if list.Entries == nil { + RespondError(w, r, "Could not resolve", http.StatusInternalServerError) + return + } + requestUri := strings.TrimPrefix(r.RequestURI, "/") + + uri, err := api.Parse(requestUri) + if err != nil { + RespondError(w, r, "Bad Request", http.StatusBadRequest) + } + + uri.Scheme = "bzz-list" + msg += fmt.Sprintf("Disambiguation:
Your request may refer to multiple choices.
Click here if your browser does not redirect you within 5 seconds.
", "/"+uri.String()) + RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices) +} + +func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { + log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) + respond(w, r, &ResponseParams{ + Code: code, + Msg: template.HTML(msg), + Timestamp: time.Now().Format(time.RFC1123), + template: TemplatesMap[templateName], + }) +} + +func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) { + log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) + RespondTemplate(w, r, "error", msg, code) +} + +func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { + + w.WriteHeader(params.Code) + + if params.Code >= 400 { + w.Header().Del("Cache-Control") + w.Header().Del("ETag") + } + + acceptHeader := r.Header.Get("Accept") + // this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8" + if strings.Contains(acceptHeader, "application/json") { + if err := respondJSON(w, r, params); err != nil { + RespondError(w, r, "Internal server error", http.StatusInternalServerError) + } + } else if strings.Contains(acceptHeader, "text/html") { + respondHTML(w, r, params) + } else { + respondPlaintext(w, r, params) //returns nice errors for curl + } +} + +func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) { + htmlCounter.Inc(1) + log.Debug("respondHTML", "ruid", GetRUID(r.Context())) + err := params.template.Execute(w, params) + if err != nil { + log.Error(err.Error()) + } +} + +func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { + jsonCounter.Inc(1) + log.Debug("respondJSON", "ruid", GetRUID(r.Context())) + w.Header().Set("Content-Type", "application/json") + return json.NewEncoder(w).Encode(params) +} + +func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { + plaintextCounter.Inc(1) + log.Debug("respondPlaintext", "ruid", GetRUID(r.Context())) + w.Header().Set("Content-Type", "text/plain") + strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n" + strToWrite += "Message: " + string(params.Msg) + "\n" + strToWrite += "Timestamp: " + params.Timestamp + "\n" + _, err := w.Write([]byte(strToWrite)) + return err +} diff --git a/swarm/api/http/error_test.go b/swarm/api/http/response_test.go similarity index 98% rename from swarm/api/http/error_test.go rename to swarm/api/http/response_test.go index 990961f60e..2e24bda6d1 100644 --- a/swarm/api/http/error_test.go +++ b/swarm/api/http/response_test.go @@ -44,7 +44,7 @@ func TestError(t *testing.T) { defer resp.Body.Close() respbody, err = ioutil.ReadAll(resp.Body) - if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") { + if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") { t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode) } diff --git a/swarm/api/http/sctx.go b/swarm/api/http/sctx.go new file mode 100644 index 0000000000..431e117354 --- /dev/null +++ b/swarm/api/http/sctx.go @@ -0,0 +1,38 @@ +package http + +import ( + "context" + + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/sctx" +) + +type contextKey int + +const ( + uriKey contextKey = iota +) + +func GetRUID(ctx context.Context) string { + v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string) + if ok { + return v + } + return "xxxxxxxx" +} + +func SetRUID(ctx context.Context, ruid string) context.Context { + return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid) +} + +func GetURI(ctx context.Context) *api.URI { + v, ok := ctx.Value(uriKey).(*api.URI) + if ok { + return v + } + return nil +} + +func SetURI(ctx context.Context, uri *api.URI) context.Context { + return context.WithValue(ctx, uriKey, uri) +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 3122654b6c..5a5c42adc0 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -41,12 +41,9 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/mru" - opentracing "github.com/opentracing/opentracing-go" - "github.com/pborman/uuid" "github.com/rs/cors" ) @@ -72,6 +69,17 @@ var ( getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) ) +type methodHandler map[string]http.Handler + +func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + v, ok := m[r.Method] + if ok { + v.ServeHTTP(rw, r) + return + } + rw.WriteHeader(http.StatusMethodNotAllowed) +} + func NewServer(api *api.API, corsString string) *Server { var allowedOrigins []string for _, domain := range strings.Split(corsString, ",") { @@ -84,20 +92,79 @@ func NewServer(api *api.API, corsString string) *Server { AllowedHeaders: []string{"*"}, }) - mux := http.NewServeMux() server := &Server{api: api} - mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz)) - mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw)) - mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable)) - mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash)) - mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList)) - mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource)) - mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths)) - mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths)) - mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths)) + defaultMiddlewares := []Adapter{ + RecoverPanic, + SetRequestID, + InitLoggingResponseWriter, + ParseURI, + InstrumentOpenTracing, + } + mux := http.NewServeMux() + mux.Handle("/bzz:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleBzzGet), + defaultMiddlewares..., + ), + "POST": Adapt( + http.HandlerFunc(server.HandlePostFiles), + defaultMiddlewares..., + ), + "DELETE": Adapt( + http.HandlerFunc(server.HandleDelete), + defaultMiddlewares..., + ), + }) + mux.Handle("/bzz-raw:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleGet), + defaultMiddlewares..., + ), + "POST": Adapt( + http.HandlerFunc(server.HandlePostRaw), + defaultMiddlewares..., + ), + }) + mux.Handle("/bzz-immutable:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleGet), + defaultMiddlewares..., + ), + }) + mux.Handle("/bzz-hash:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleGet), + defaultMiddlewares..., + ), + }) + mux.Handle("/bzz-list:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleGetList), + defaultMiddlewares..., + ), + }) + mux.Handle("/bzz-resource:/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleGetResource), + defaultMiddlewares..., + ), + "POST": Adapt( + http.HandlerFunc(server.HandlePostResource), + defaultMiddlewares..., + ), + }) + + mux.Handle("/", methodHandler{ + "GET": Adapt( + http.HandlerFunc(server.HandleRootPaths), + SetRequestID, + InitLoggingResponseWriter, + ), + }) server.Handler = c.Handler(mux) + return server } @@ -105,139 +172,6 @@ func (s *Server) ListenAndServe(addr string) error { return http.ListenAndServe(addr, s) } -func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - if r.RequestURI == "/" { - if strings.Contains(r.Header.Get("Accept"), "text/html") { - err := landingPageTemplate.Execute(w, nil) - if err != nil { - log.Error(fmt.Sprintf("error rendering landing page: %s", err)) - } - return - } - if strings.Contains(r.Header.Get("Accept"), "application/json") { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode("Welcome to Swarm!") - return - } - } - - if r.URL.Path == "/robots.txt" { - w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) - fmt.Fprintf(w, "User-agent: *\nDisallow: /") - return - } - Respond(w, r, "Bad Request", http.StatusBadRequest) - default: - Respond(w, r, "Not Found", http.StatusNotFound) - } -} - -func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - log.Debug("handleGetBzz") - if r.Header.Get("Accept") == "application/x-tar" { - reader, err := s.api.GetDirectoryTar(r.Context(), r.uri) - if err != nil { - Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) - } - defer reader.Close() - - w.Header().Set("Content-Type", "application/x-tar") - w.WriteHeader(http.StatusOK) - io.Copy(w, reader) - return - } - s.HandleGetFile(w, r) - case http.MethodPost: - log.Debug("handlePostFiles") - s.HandlePostFiles(w, r) - case http.MethodDelete: - log.Debug("handleBzzDelete") - s.HandleDelete(w, r) - default: - Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) - } -} -func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - log.Debug("handleGetRaw") - s.HandleGet(w, r) - case http.MethodPost: - log.Debug("handlePostRaw") - s.HandlePostRaw(w, r) - default: - Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) - } -} -func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - log.Debug("handleGetHash") - s.HandleGetList(w, r) - default: - Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) - } -} -func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - log.Debug("handleGetHash") - s.HandleGet(w, r) - default: - Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) - } -} -func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - log.Debug("handleGetHash") - s.HandleGetList(w, r) - default: - Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) - } -} -func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) { - switch r.Method { - case http.MethodGet: - log.Debug("handleGetResource") - s.HandleGetResource(w, r) - case http.MethodPost: - log.Debug("handlePostResource") - s.HandlePostResource(w, r) - default: - Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) - } -} -func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc { - return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now()) - req := &Request{Request: *r, ruid: uuid.New()[:8]} - metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1) - log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI) - - // wrapping the ResponseWriter, so that we get the response code set by http.ServeContent - w := newLoggingResponseWriter(rw) - if parseBzzUri { - uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) - if err != nil { - Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) - return - } - req.uri = uri - - log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme) - } - - h(w, req) // call original - log.Info("served response", "ruid", req.ruid, "code", w.statusCode) - }) -} - // browser API for registering bzz url scheme handlers: // https://developer.mozilla.org/en/docs/Web-based_protocol_handlers // electron (chromium) api for registering bzz url scheme handlers: @@ -247,59 +181,81 @@ type Server struct { api *api.API } -// Request wraps http.Request and also includes the parsed bzz URI -type Request struct { - http.Request +func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { + log.Debug("handleBzzGet", "ruid", GetRUID(r.Context())) + if r.Header.Get("Accept") == "application/x-tar" { + uri := GetURI(r.Context()) + reader, err := s.api.GetDirectoryTar(r.Context(), uri) + if err != nil { + RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + } + defer reader.Close() - uri *api.URI - ruid string // request unique id + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(http.StatusOK) + io.Copy(w, reader) + return + } + + s.HandleGetFile(w, r) +} + +func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/": + RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) + return + case "/robots.txt": + w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) + fmt.Fprintf(w, "User-agent: *\nDisallow: /") + case "/favicon.ico": + w.WriteHeader(http.StatusOK) + w.Write(faviconBytes) + default: + RespondError(w, r, "Not Found", http.StatusNotFound) + } } // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request // body in swarm and returns the resulting storage address as a text/plain response -func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { - log.Debug("handle.post.raw", "ruid", r.ruid) +func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + log.Debug("handle.post.raw", "ruid", ruid) postRawCount.Inc(1) - ctx := r.Context() - var sp opentracing.Span - ctx, sp = spancontext.StartSpan( - ctx, - "http.post.raw") - defer sp.Finish() - toEncrypt := false - if r.uri.Addr == "encrypt" { + uri := GetURI(r.Context()) + if uri.Addr == "encrypt" { toEncrypt = true } - if r.uri.Path != "" { + if uri.Path != "" { postRawFail.Inc(1) - Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) + RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) return } - if r.uri.Addr != "" && r.uri.Addr != "encrypt" { + if uri.Addr != "" && uri.Addr != "encrypt" { postRawFail.Inc(1) - Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) + RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { postRawFail.Inc(1) - Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest) + RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } - addr, _, err := s.api.Store(ctx, r.Body, r.ContentLength, toEncrypt) + addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) return } - log.Debug("stored content", "ruid", r.ruid, "key", addr) + log.Debug("stored content", "ruid", ruid, "key", addr) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) @@ -311,55 +267,49 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { // (either a tar archive or multipart form), adds those files either to an // existing manifest or to a new manifest under and returns the // resulting manifest hash as a text/plain response -func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { - log.Debug("handle.post.files", "ruid", r.ruid) +func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + log.Debug("handle.post.files", "ruid", ruid) postFilesCount.Inc(1) - var sp opentracing.Span - ctx := r.Context() - ctx, sp = spancontext.StartSpan( - ctx, - "http.post.files") - defer sp.Finish() - contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { postFilesFail.Inc(1) - Respond(w, r, err.Error(), http.StatusBadRequest) + RespondError(w, r, err.Error(), http.StatusBadRequest) return } toEncrypt := false - if r.uri.Addr == "encrypt" { + uri := GetURI(r.Context()) + if uri.Addr == "encrypt" { toEncrypt = true } var addr storage.Address - if r.uri.Addr != "" && r.uri.Addr != "encrypt" { - addr, err = s.api.Resolve(r.Context(), r.uri) + if uri.Addr != "" && uri.Addr != "encrypt" { + addr, err = s.api.Resolve(r.Context(), uri) if err != nil { postFilesFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) return } - log.Debug("resolved key", "ruid", r.ruid, "key", addr) + log.Debug("resolved key", "ruid", ruid, "key", addr) } else { addr, err = s.api.NewManifest(r.Context(), toEncrypt) if err != nil { postFilesFail.Inc(1) - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) return } - log.Debug("new manifest", "ruid", r.ruid, "key", addr) + log.Debug("new manifest", "ruid", ruid, "key", addr) } - newAddr, err := s.api.UpdateManifest(ctx, addr, func(mw *api.ManifestWriter) error { + newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error { switch contentType { - case "application/x-tar": _, err := s.handleTarUpload(r, mw) if err != nil { - Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) return err } return nil @@ -372,30 +322,33 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { }) if err != nil { postFilesFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } - log.Debug("stored content", "ruid", r.ruid, "key", newAddr) + log.Debug("stored content", "ruid", ruid, "key", newAddr) w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) fmt.Fprint(w, newAddr) } -func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) { - log.Debug("handle.tar.upload", "ruid", r.ruid) +func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) { + log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context())) - key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw) + defaultPath := r.URL.Query().Get("defaultpath") + + key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, defaultPath, mw) if err != nil { return nil, err } return key, nil } -func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error { - log.Debug("handle.multipart.upload", "ruid", req.ruid) - mr := multipart.NewReader(req.Body, boundary) +func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api.ManifestWriter) error { + ruid := GetRUID(r.Context()) + log.Debug("handle.multipart.upload", "ruid", ruid) + mr := multipart.NewReader(r.Body, boundary) for { part, err := mr.NextPart() if err == io.EOF { @@ -435,48 +388,52 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma if name == "" { name = part.FormName() } - path := path.Join(req.uri.Path, name) + uri := GetURI(r.Context()) + path := path.Join(uri.Path, name) entry := &api.ManifestEntry{ Path: path, ContentType: part.Header.Get("Content-Type"), Size: size, ModTime: time.Now(), } - log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path) - contentKey, err := mw.AddEntry(req.Context(), reader, entry) + log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path) + contentKey, err := mw.AddEntry(r.Context(), reader, entry) if err != nil { return fmt.Errorf("error adding manifest entry from multipart form: %s", err) } - log.Debug("stored content", "ruid", req.ruid, "key", contentKey) + log.Debug("stored content", "ruid", ruid, "key", contentKey) } } -func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error { - log.Debug("handle.direct.upload", "ruid", req.ruid) - key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{ - Path: req.uri.Path, - ContentType: req.Header.Get("Content-Type"), +func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) error { + ruid := GetRUID(r.Context()) + log.Debug("handle.direct.upload", "ruid", ruid) + key, err := mw.AddEntry(r.Context(), r.Body, &api.ManifestEntry{ + Path: GetURI(r.Context()).Path, + ContentType: r.Header.Get("Content-Type"), Mode: 0644, - Size: req.ContentLength, + Size: r.ContentLength, ModTime: time.Now(), }) if err != nil { return err } - log.Debug("stored content", "ruid", req.ruid, "key", key) + log.Debug("stored content", "ruid", ruid, "key", key) return nil } // HandleDelete handles a DELETE request to bzz://, removes // from and returns the resulting manifest hash as a // text/plain response -func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { - log.Debug("handle.delete", "ruid", r.ruid) +func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + uri := GetURI(r.Context()) + log.Debug("handle.delete", "ruid", ruid) deleteCount.Inc(1) - newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path) + newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path) if err != nil { deleteFail.Inc(1) - Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) return } @@ -519,27 +476,20 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) { // // The POST request admits a JSON structure as defined in the mru package: `mru.updateRequestJSON` // The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content -func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { - log.Debug("handle.post.resource", "ruid", r.ruid) - - var sp opentracing.Span - ctx := r.Context() - ctx, sp = spancontext.StartSpan( - ctx, - "http.post.resource") - defer sp.Finish() - +func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + log.Debug("handle.post.resource", "ruid", ruid) var err error // Creation and update must send mru.updateRequestJSON JSON structure body, err := ioutil.ReadAll(r.Body) if err != nil { - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) return } var updateRequest mru.Request if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON - Respond(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error + RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error return } @@ -548,7 +498,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { // to update this resource // Check this early, to avoid creating a resource and then not being able to set its first update. if err = updateRequest.Verify(); err != nil { - Respond(w, r, err.Error(), http.StatusForbidden) + RespondError(w, r, err.Error(), http.StatusForbidden) return } } @@ -557,7 +507,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { err = s.api.ResourceCreate(r.Context(), &updateRequest) if err != nil { code, err2 := s.translateResourceError(w, r, "resource creation fail", err) - Respond(w, r, err2.Error(), code) + RespondError(w, r, err2.Error(), code) return } } @@ -565,7 +515,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { if updateRequest.IsUpdate() { _, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate) if err != nil { - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) return } } @@ -579,7 +529,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { // metadata chunk (rootAddr) m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex()) if err != nil { - Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) return } @@ -589,7 +539,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { // \TODO update manifest key automatically in ENS outdata, err := json.Marshal(m) if err != nil { - Respond(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) return } fmt.Fprint(w, string(outdata)) @@ -604,17 +554,19 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { // bzz-resource:///meta - get metadata and next version information // = ens name or hash // TODO: Enable pass maxPeriod parameter -func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { - log.Debug("handle.get.resource", "ruid", r.ruid) +func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + uri := GetURI(r.Context()) + log.Debug("handle.get.resource", "ruid", ruid) var err error // resolve the content key. - manifestAddr := r.uri.Address() + manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), r.uri) + manifestAddr, err = s.api.Resolve(r.Context(), uri) if err != nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } } else { @@ -625,25 +577,25 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr) if err != nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound) return } - log.Debug("handle.get.resource: resolved", "ruid", r.ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr) + log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr) // determine if the query specifies period and version or it is a metadata query var params []string - if len(r.uri.Path) > 0 { - if r.uri.Path == "meta" { + if len(uri.Path) > 0 { + if uri.Path == "meta" { unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr) if err != nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound) return } rawResponse, err := unsignedUpdateRequest.MarshalJSON() if err != nil { - Respond(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError) return } w.Header().Add("Content-type", "application/json") @@ -653,7 +605,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { } - params = strings.Split(r.uri.Path, "/") + params = strings.Split(uri.Path, "/") } var name string @@ -689,17 +641,17 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { // any error from the switch statement will end up here if err != nil { code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err) - Respond(w, r, err2.Error(), code) + RespondError(w, r, err2.Error(), code) return } // All ok, serve the retrieved update - log.Debug("Found update", "name", name, "ruid", r.ruid) + log.Debug("Found update", "name", name, "ruid", ruid) w.Header().Set("Content-Type", "application/octet-stream") - http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) + http.ServeContent(w, r, "", now, bytes.NewReader(data)) } -func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) (int, error) { +func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) { code := 0 defaultErr := fmt.Errorf("%s: %v", supErr, err) rsrcErr, ok := err.(*mru.Error) @@ -725,46 +677,41 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr // given storage key // - bzz-hash:// and responds with the hash of the content stored // at the given storage key as a text/plain response -func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { - log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri) +func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + uri := GetURI(r.Context()) + log.Debug("handle.get", "ruid", ruid, "uri", uri) getCount.Inc(1) - var sp opentracing.Span - ctx := r.Context() - ctx, sp = spancontext.StartSpan( - ctx, - "http.get") - defer sp.Finish() - var err error - addr := r.uri.Address() + addr := uri.Address() if addr == nil { - addr, err = s.api.Resolve(r.Context(), r.uri) + addr, err = s.api.Resolve(r.Context(), uri) if err != nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } } else { w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. } - log.Debug("handle.get: resolved", "ruid", r.ruid, "key", addr) + log.Debug("handle.get: resolved", "ruid", ruid, "key", addr) // if path is set, interpret as a manifest and return the // raw entry at the given path - if r.uri.Path != "" { + if uri.Path != "" { walker, err := s.api.NewManifestWalker(r.Context(), addr, nil) if err != nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest) + RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest) return } var entry *api.ManifestEntry walker.Walk(func(e *api.ManifestEntry) error { // if the entry matches the path, set entry and stop // the walk - if e.Path == r.uri.Path { + if e.Path == uri.Path { entry = e // return an error to cancel the walk return errors.New("found") @@ -778,7 +725,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // if the manifest's path is a prefix of the // requested path, recurse into it by returning // nil and continuing the walk - if strings.HasPrefix(r.uri.Path, e.Path) { + if strings.HasPrefix(uri.Path, e.Path) { return nil } @@ -786,7 +733,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { }) if entry == nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound) return } addr = storage.Address(common.Hex2Bytes(entry.Hash)) @@ -796,23 +743,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key. if noneMatchEtag != "" { if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) { - Respond(w, r, "Not Modified", http.StatusNotModified) + w.WriteHeader(http.StatusNotModified) return } } // check the root chunk exists by retrieving the file's size - reader, isEncrypted := s.api.Retrieve(ctx, addr) - if _, err := reader.Size(ctx, nil); err != nil { + reader, isEncrypted := s.api.Retrieve(r.Context(), addr) + if _, err := reader.Size(r.Context(), nil); err != nil { getFail.Inc(1) - Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) return } w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted)) switch { - case r.uri.Raw(): + case uri.Raw(): // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" @@ -820,8 +767,8 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { contentType = typ } w.Header().Set("Content-Type", contentType) - http.ServeContent(w, &r.Request, "", time.Now(), reader) - case r.uri.Hash(): + http.ServeContent(w, r, "", time.Now(), reader) + case uri.Hash(): w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) fmt.Fprint(w, addr) @@ -831,35 +778,30 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // HandleGetList handles a GET request to bzz-list:// and returns // a list of all files contained in under grouped into // common prefixes using "/" as a delimiter -func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { - log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri) +func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + uri := GetURI(r.Context()) + log.Debug("handle.get.list", "ruid", ruid, "uri", uri) getListCount.Inc(1) - var sp opentracing.Span - ctx := r.Context() - ctx, sp = spancontext.StartSpan( - ctx, - "http.get.list") - defer sp.Finish() - // ensure the root path has a trailing slash so that relative URLs work - if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { - http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) + if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { + http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently) return } - addr, err := s.api.Resolve(r.Context(), r.uri) + addr, err := s.api.Resolve(r.Context(), uri) if err != nil { getListFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } - log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr) + log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr) - list, err := s.api.GetManifestList(ctx, addr, r.uri.Path) + list, err := s.api.GetManifestList(r.Context(), addr, uri.Path) if err != nil { getListFail.Inc(1) - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -867,11 +809,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { // HTML index with relative URLs if strings.Contains(r.Header.Get("Accept"), "text/html") { w.Header().Set("Content-Type", "text/html") - err := htmlListTemplate.Execute(w, &htmlListData{ + err := TemplatesMap["bzz-list"].Execute(w, &htmlListData{ URI: &api.URI{ Scheme: "bzz", - Addr: r.uri.Addr, - Path: r.uri.Path, + Addr: uri.Addr, + Path: uri.Path, }, List: &list, }) @@ -888,45 +830,40 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { // HandleGetFile handles a GET request to bzz:/// and responds // with the content of the file at from the given -func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { - log.Debug("handle.get.file", "ruid", r.ruid) +func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { + ruid := GetRUID(r.Context()) + uri := GetURI(r.Context()) + log.Debug("handle.get.file", "ruid", ruid) getFileCount.Inc(1) - var sp opentracing.Span - ctx := r.Context() - ctx, sp = spancontext.StartSpan( - ctx, - "http.get.file") - defer sp.Finish() - // ensure the root path has a trailing slash so that relative URLs work - if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { - http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) + if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { + http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently) return } var err error - manifestAddr := r.uri.Address() + manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), r.uri) + manifestAddr, err = s.api.Resolve(r.Context(), uri) if err != nil { getFileFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } } else { w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. } - log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr) - reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path) + log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr) + reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path) etag := common.Bytes2Hex(contentKey) noneMatchEtag := r.Header.Get("If-None-Match") w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key. if noneMatchEtag != "" { if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) { - Respond(w, r, "Not Modified", http.StatusNotModified) + w.WriteHeader(http.StatusNotModified) return } } @@ -935,10 +872,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { switch status { case http.StatusNotFound: getFileNotFound.Inc(1) - Respond(w, r, err.Error(), http.StatusNotFound) + RespondError(w, r, err.Error(), http.StatusNotFound) default: getFileFail.Inc(1) - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) } return } @@ -946,28 +883,28 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { //the request results in ambiguous files //e.g. /read with readme.md and readinglist.txt available in manifest if status == http.StatusMultipleChoices { - list, err := s.api.GetManifestList(ctx, manifestAddr, r.uri.Path) + list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path) if err != nil { getFileFail.Inc(1) - Respond(w, r, err.Error(), http.StatusInternalServerError) + RespondError(w, r, err.Error(), http.StatusInternalServerError) return } - log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", r.ruid) + log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", ruid) //show a nice page links to available entries ShowMultipleChoices(w, r, list) return } // check the root chunk exists by retrieving the file's size - if _, err := reader.Size(ctx, nil); err != nil { + if _, err := reader.Size(r.Context(), nil); err != nil { getFileNotFound.Inc(1) - Respond(w, r, fmt.Sprintf("file not found %s: %s", r.uri, err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) return } w.Header().Set("Content-Type", contentType) - http.ServeContent(w, &r.Request, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) + http.ServeContent(w, r, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) } // The size of buffer used for bufio.Reader on LazyChunkReader passed to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index e8bc1bdaea..7934e37eba 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -27,8 +27,10 @@ import ( "fmt" "io" "io/ioutil" + "mime/multipart" "net/http" "os" + "strconv" "strings" "testing" "time" @@ -443,11 +445,6 @@ func TestBzzGetPath(t *testing.T) { testBzzGetPath(true, t) } -func TestBzzTar(t *testing.T) { - testBzzTar(false, t) - testBzzTar(true, t) -} - func testBzzGetPath(encrypted bool, t *testing.T) { var err error @@ -561,24 +558,35 @@ func testBzzGetPath(encrypted bool, t *testing.T) { ref := addr[2].Hex() for _, c := range []struct { - path string - json string - html string + path string + json string + pageFragments []string }{ { path: "/", json: `{"common_prefixes":["a/"]}`, - html: fmt.Sprintf("\n\n\n \n \n\t\t\n\tSwarm index of bzz:/%s/\n\n\n\n

Swarm index of bzz:/%s/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n
PathTypeSize
a/DIR-
\n
\n\n", ref, ref), + pageFragments: []string{ + fmt.Sprintf("Swarm index of bzz:/%s/", ref), + `a/`, + }, }, { path: "/a/", json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`, - html: fmt.Sprintf("\n\n\n \n \n\t\t\n\tSwarm index of bzz:/%s/a/\n\n\n\n

Swarm index of bzz:/%s/a/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b/DIR-
a0
\n
\n\n", ref, ref), + pageFragments: []string{ + fmt.Sprintf("Swarm index of bzz:/%s/a/", ref), + `b/`, + fmt.Sprintf(`a`, ref), + }, }, { path: "/a/b/", json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`, - html: fmt.Sprintf("\n\n\n \n \n\t\t\n\tSwarm index of bzz:/%s/a/b/\n\n\n\n

Swarm index of bzz:/%s/a/b/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\n \n\t\n\t \n\t \n\t \n\t\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b0
c0
\n
\n\n", ref, ref), + pageFragments: []string{ + fmt.Sprintf("Swarm index of bzz:/%s/a/b/", ref), + fmt.Sprintf(`b`, ref), + fmt.Sprintf(`c`, ref), + }, }, { path: "/x", @@ -628,21 +636,25 @@ func testBzzGetPath(encrypted bool, t *testing.T) { t.Fatalf("HTTP request: %v", err) } defer resp.Body.Close() - respbody, err := ioutil.ReadAll(resp.Body) + b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatalf("Read response body: %v", err) } - if string(respbody) != c.html { - isexpectedfailrequest := false + body := string(b) - for _, r := range expectedfailrequests { - if k[:] == r { - isexpectedfailrequest = true + for _, f := range c.pageFragments { + if !strings.Contains(body, f) { + isexpectedfailrequest := false + + for _, r := range expectedfailrequests { + if k[:] == r { + isexpectedfailrequest = true + } + } + if !isexpectedfailrequest { + t.Errorf("Response list body %q does not contain %q: body %q", k, f, body) } - } - if !isexpectedfailrequest { - t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody)) } } }) @@ -657,11 +669,11 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } nonhashresponses := []string{ - "cannot resolve name: no DNS to resolve name: "name"", - "cannot resolve nonhash: immutable address not a content hash: "nonhash"", - "cannot resolve nonhash: no DNS to resolve name: "nonhash"", - "cannot resolve nonhash: no DNS to resolve name: "nonhash"", - "cannot resolve nonhash: no DNS to resolve name: "nonhash"", + `cannot resolve name: no DNS to resolve name: "name"`, + `cannot resolve nonhash: immutable address not a content hash: "nonhash"`, + `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, + `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, + `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, } for i, url := range nonhashtests { @@ -684,6 +696,11 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } } +func TestBzzTar(t *testing.T) { + testBzzTar(false, t) + testBzzTar(true, t) +} + func testBzzTar(encrypted bool, t *testing.T) { srv := testutil.NewTestSwarmServer(t, serverFunc) defer srv.Close() @@ -738,7 +755,6 @@ func testBzzTar(encrypted bool, t *testing.T) { } swarmHash, err := ioutil.ReadAll(resp2.Body) resp2.Body.Close() - t.Logf("uploaded tarball successfully and got manifest address at %s", string(swarmHash)) if err != nil { t.Fatal(err) } @@ -887,8 +903,253 @@ func TestMethodsNotAllowed(t *testing.T) { } { res, _ := http.Post(c.url, "text/plain", bytes.NewReader([]byte(databytes))) if res.StatusCode != c.code { - t.Fatal("should have failed") + t.Fatalf("should have failed. requested url: %s, expected code %d, got %d", c.url, c.code, res.StatusCode) } } } + +func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string]string, verbose bool, t *testing.T) (*http.Response, string) { + // Build the Request + req, err := http.NewRequest(httpMethod, url, reqBody) + if err != nil { + t.Fatal(err) + } + for key, value := range headers { + req.Header.Set(key, value) + } + if verbose { + t.Log(req.Method, req.URL, req.Header, req.Body) + } + + // Send Request out + httpClient := &http.Client{} + res, err := httpClient.Do(req) + if err != nil { + t.Fatal(err) + } + + // Read the HTTP Body + buffer, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + body := string(buffer) + + return res, body +} + +func TestGet(t *testing.T) { + srv := testutil.NewTestSwarmServer(t, serverFunc) + defer srv.Close() + + for _, testCase := range []struct { + uri string + method string + headers map[string]string + expectedStatusCode int + assertResponseBody string + verbose bool + }{ + { + uri: fmt.Sprintf("%s/", srv.URL), + method: "GET", + headers: map[string]string{"Accept": "text/html"}, + expectedStatusCode: 200, + assertResponseBody: "Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution", + verbose: false, + }, + { + uri: fmt.Sprintf("%s/", srv.URL), + method: "GET", + headers: map[string]string{"Accept": "application/json"}, + expectedStatusCode: 200, + assertResponseBody: "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", + verbose: false, + }, + { + uri: fmt.Sprintf("%s/robots.txt", srv.URL), + method: "GET", + headers: map[string]string{"Accept": "text/html"}, + expectedStatusCode: 200, + assertResponseBody: "User-agent: *\nDisallow: /", + verbose: false, + }, + { + uri: fmt.Sprintf("%s/nonexistent_path", srv.URL), + method: "GET", + headers: map[string]string{}, + expectedStatusCode: 404, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz:asdf/", srv.URL), + method: "GET", + headers: map[string]string{}, + expectedStatusCode: 404, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/tbz2/", srv.URL), + method: "GET", + headers: map[string]string{}, + expectedStatusCode: 404, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz-rack:/", srv.URL), + method: "GET", + headers: map[string]string{}, + expectedStatusCode: 404, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz-ls", srv.URL), + method: "GET", + headers: map[string]string{}, + expectedStatusCode: 404, + verbose: false, + }, + } { + t.Run("GET "+testCase.uri, func(t *testing.T) { + res, body := httpDo(testCase.method, testCase.uri, nil, testCase.headers, testCase.verbose, t) + if res.StatusCode != testCase.expectedStatusCode { + t.Fatalf("expected status code %d but got %d", testCase.expectedStatusCode, res.StatusCode) + } + if testCase.assertResponseBody != "" && !strings.Contains(body, testCase.assertResponseBody) { + t.Fatalf("expected response to be: %s but got: %s", testCase.assertResponseBody, body) + } + }) + } +} + +func TestModify(t *testing.T) { + srv := testutil.NewTestSwarmServer(t, serverFunc) + defer srv.Close() + + swarmClient := swarm.NewClient(srv.URL) + data := []byte("data") + file := &swarm.File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), + ManifestEntry: api.ManifestEntry{ + Path: "", + ContentType: "text/plain", + Size: int64(len(data)), + }, + } + + hash, err := swarmClient.Upload(file, "", false) + if err != nil { + t.Fatal(err) + } + + for _, testCase := range []struct { + uri string + method string + headers map[string]string + requestBody []byte + expectedStatusCode int + assertResponseBody string + assertResponseHeaders map[string]string + verbose bool + }{ + { + uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), + method: "DELETE", + headers: map[string]string{}, + expectedStatusCode: 200, + assertResponseBody: "8b634aea26eec353ac0ecbec20c94f44d6f8d11f38d4578a4c207a84c74ef731", + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), + method: "PUT", + headers: map[string]string{}, + expectedStatusCode: 405, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, hash), + method: "PUT", + headers: map[string]string{}, + expectedStatusCode: 405, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), + method: "PATCH", + headers: map[string]string{}, + expectedStatusCode: 405, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz-raw:/", srv.URL), + method: "POST", + headers: map[string]string{}, + requestBody: []byte("POSTdata"), + expectedStatusCode: 200, + assertResponseHeaders: map[string]string{"Content-Length": "64"}, + verbose: false, + }, + { + uri: fmt.Sprintf("%s/bzz-raw:/encrypt", srv.URL), + method: "POST", + headers: map[string]string{}, + requestBody: []byte("POSTdata"), + expectedStatusCode: 200, + assertResponseHeaders: map[string]string{"Content-Length": "128"}, + verbose: false, + }, + } { + t.Run(testCase.method+" "+testCase.uri, func(t *testing.T) { + reqBody := bytes.NewReader(testCase.requestBody) + res, body := httpDo(testCase.method, testCase.uri, reqBody, testCase.headers, testCase.verbose, t) + + if res.StatusCode != testCase.expectedStatusCode { + t.Fatalf("expected status code %d but got %d", testCase.expectedStatusCode, res.StatusCode) + } + if testCase.assertResponseBody != "" && !strings.Contains(body, testCase.assertResponseBody) { + t.Log(body) + t.Fatalf("expected response %s but got %s", testCase.assertResponseBody, body) + } + for key, value := range testCase.assertResponseHeaders { + if res.Header.Get(key) != value { + t.Logf("expected %s=%s in HTTP response header but got %s", key, value, res.Header.Get(key)) + } + } + }) + } +} + +func TestMultiPartUpload(t *testing.T) { + // POST /bzz:/ Content-Type: multipart/form-data + verbose := false + // Setup Swarm + srv := testutil.NewTestSwarmServer(t, serverFunc) + defer srv.Close() + + url := fmt.Sprintf("%s/bzz:/", srv.URL) + + buf := new(bytes.Buffer) + form := multipart.NewWriter(buf) + form.WriteField("name", "John Doe") + file1, _ := form.CreateFormFile("cv", "cv.txt") + file1.Write([]byte("John Doe's Credentials")) + file2, _ := form.CreateFormFile("profile_picture", "profile.jpg") + file2.Write([]byte("imaginethisisjpegdata")) + form.Close() + + headers := map[string]string{ + "Content-Type": form.FormDataContentType(), + "Content-Length": strconv.Itoa(buf.Len()), + } + res, body := httpDo("POST", url, buf, headers, verbose, t) + + if res.StatusCode != 200 { + t.Fatalf("expected POST multipart/form-data to return 200, but it returned %d", res.StatusCode) + } + if len(body) != 64 { + t.Fatalf("expected POST multipart/form-data to return a 64 char manifest but the answer was %d chars long", len(body)) + } +} diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go index 8897b96946..986f5f8873 100644 --- a/swarm/api/http/templates.go +++ b/swarm/api/http/templates.go @@ -17,6 +17,8 @@ package http import ( + "encoding/hex" + "fmt" "html/template" "path" @@ -28,178 +30,277 @@ type htmlListData struct { List *api.ManifestList } -var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(` - - - - - - - Swarm index of {{ .URI }} - +var TemplatesMap = make(map[string]*template.Template) +var faviconBytes []byte - -

Swarm index of {{ .URI }}

-
- - - - - - - - +func init() { + for _, v := range []struct { + templateName string + partial string + funcs template.FuncMap + }{ + { + templateName: "error", + partial: errorResponse, + }, + { + templateName: "bzz-list", + partial: bzzList, + funcs: template.FuncMap{ + "basename": path.Base, + "leaflink": leafLink, + }, + }, + { + templateName: "landing-page", + partial: landing, + }, + } { + TemplatesMap[v.templateName] = template.Must(template.New(v.templateName).Funcs(v.funcs).Parse(baseTemplate + css + v.partial + logo)) + } - - {{ range .List.CommonPrefixes }} - - - - - - {{ end }} + bytes, err := hex.DecodeString(favicon) + if err != nil { + panic(err) + } + faviconBytes = bytes +} - {{ range .List.Entries }} - - - - - - {{ end }} -
PathTypeSize
{{ basename . }}/DIR-
{{ basename .Path }}{{ .ContentType }}{{ .Size }}
-
- -`[1:])) +func leafLink(URI api.URI, manifestEntry api.ManifestEntry) string { + return fmt.Sprintf("/bzz:/%s/%s", URI.Addr, manifestEntry.Path) +} -var landingPageTemplate = template.Must(template.New("landingPage").Parse(` - - - - - - - - Swarm :: Welcome to Swarm - - - -
-
- -
-
-

Welcome to Swarm

-
-
- - - - -

Enter the hash or ENS of a Swarm-hosted file below:

-
- - -
-
-
-
-

- Swarm: Serverless Hosting Incentivised peer-to-peer Storage and Content Distribution -

-
+ + + - +{{ end }}` + +const baseTemplate = ` + + + + + + + + + {{ template "content" . }} + -`[1:])) +` + +const css = `{{ define "css" }} +html { + font-size: 18px; + font-size: 1.13rem; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; + font-family: Helvetica, Arial, sans-serif; +} + +body { + background: #f6f6f6; + color: #333; +} + +a, a:visited, a:active { + color: darkorange; +} + +a.normal-link, a.normal-link:active { color: #0000EE; } +a.normal-link:visited { color: #551A8B; } + +table { + border-collapse: separate; +} + +td { + padding: 3px 10px; +} + + +.container { + max-width: 600px; + margin: 40px auto 40px; + text-align: center; +} + +.separate-block { + margin: 40px 0; + word-wrap: break-word; +} + +.footer { + font-size: 12px; + font-size: 0.75rem; + text-align: center; +} + +.orange { + color: #ffa500; +} + +.top-space { + margin-top: 20px; + margin-bottom: 20px; +} + +/* SVG Logos, editable */ + +.searchbar { + padding: 20px 20px 0; +} + +.logo { + margin: 100px 80px 0; +} + +.logo a img { + max-width: 140px; +} + +/* Tablet < 600p*/ + +@media only screen and (max-width: 600px) {} + +/* Mobile phone < 360p*/ + +@media only screen and (max-width: 360px) { + h1 { + font-size: 20px; + font-size: 1.5rem; + } + h2 { + font-size: 0.88rem; + margin: 0; + } + .logo { + margin: 50px 40px 0; + } + .footer { + font-size: 0.63rem; + text-align: center; + } +} + +input[type=text] { + width: 100%; + box-sizing: border-box; + border: 2px solid #777; + border-radius: 2px; + font-size: 16px; + padding: 12px 20px 12px 20px; + transition: border 250ms ease-in-out; +} + +input[type=text]:focus { + border: 2px solid #ffce73; +} + +.button { + background-color: #ffa500; + margin: 20px 0; + border: none; + border-radius: 2px; + color: #222; + padding: 15px 32px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 16px; +} +{{ end }}` + +const logo = `{{ define "logo" }} + +{{ end }}` + +const favicon = `000001000400101000000000200068040000460000002020000000002000a8100000ae0400003030000000002000a825000056150000404000000000200028420000fe3a000028000000100000002000000001002000000000004004000000000000000000000000000000000000ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017e7e7e0362626263545454c548484849ffffff01ffffff01ffffff01ffffff01646464375b5b5bbf4545457758585809ffffff01ffffff01ffffff0164646443626262cf626262ff535353ff454545ff454545b74949492b6868681d626262a5626262fd5c5c5cff464646ff454545dd47474755ffffff01ffffff013f3f3feb565656ff636363ff535353ff464646ff3f3f3fff373737ab393939894d4d4dff626262ff5c5c5cff464646ff424242ff3a3a3af7ffffff01ffffff01383838e9353535ff424242ff474747ff383838ff353535ff363636ab35353587363636ff3a3a3aff4a4a4aff3b3b3bff353535ff363636f5ffffff01ffffff01383838e9303030ff181818ff131313ff232323ff343434ff363636ab35353587343434ff202020ff101010ff1d1d1dff303030ff373737f5ffffff01ffffff01232323c50c0c0cff0d0d0dff131313ff171717ff171717ff2929298b2727276b0f0f0ffd0d0d0dff101010ff171717ff161616ff232323d9ffffff01ffffff014d4d4d030f0f0f650c0c0ce7131313ff161616d51d1d1d4b63636363464646691717173b0d0d0dc50f0f0fff161616ef171717752e2e2e07ffffff01ffffff01ffffff01ffffff011d1d1d0f1515155360606045626262cf636363ff464646ff454545d3484848491414144d24242417ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013c3c3c374f4f4fff636363ff636363ff464646ff464646ff3f3f3fff3c3c3c41ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013636363d353535ff3c3c3cff575757ff363636ff181818ff282828ff37373747ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013636363d363636ff303030ff181818ff292929ff131313ef17171771696969136565653bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01323232371e1e1eff0d0d0dff0c0c0cff363636ff363636a3ffffff0185858515606060ff4747476bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01111111450d0d0dd10c0c0cff1b1b1bff2a2a2a993e3e3e0b30303085292929ff37373787ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636030e0e0e671616166b45454505323232432e2e2ed9151515c31d1d1d2dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014e4e4e05ffffff01ffffff01ffffff01ffffff010000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff28000000200000004000000001002000000000008010000000000000000000000000000000000000ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017272721b646464a54646466f72727205ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0168686845575757b74f4f4f39ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017e7e7e0b6262627d616161f3636363ff424242ff444444d74f4f4f49ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff016c6c6c27636363b5616161ff555555ff434343ff464646a35858581dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff016666665d616161e3626262ff636363ff636363ff444444ff464646ff434343ff454545b95252522bffffff01ffffff01ffffff01ffffff016c6c6c1363636393616161fb636363ff636363ff555555ff464646ff464646ff444444f5464646836666660bffffff01ffffff01ffffff01ffffff01ffffff016a6a6a3f626262c9616161ff636363ff636363ff636363ff636363ff444444ff464646ff464646ff464646ff434343fb48484897545454135b5b5b036868686f616161ef626262ff636363ff636363ff636363ff555555ff464646ff464646ff464646ff454545ff444444e54a4a4a5fffffff01ffffff01ffffff01ffffff013b3b3bd7505050ff646464ff636363ff636363ff636363ff636363ff444444ff464646ff464646ff464646ff454545ff3a3a3aff33333357313131113c3c3cff5a5a5aff646464ff636363ff636363ff636363ff555555ff464646ff464646ff464646ff464646ff424242ff383838f1ffffff01ffffff01ffffff01ffffff013a3a3ad5353535ff3a3a3aff575757ff646464ff626262ff636363ff444444ff464646ff464646ff3d3d3dff353535ff363636ff3636365535353511363636ff343434ff434343ff606060ff636363ff636363ff555555ff464646ff464646ff444444ff393939ff353535ff373737edffffff01ffffff01ffffff01ffffff013a3a3ad5363636ff363636ff343434ff3f3f3fff5d5d5dff646464ff444444ff404040ff363636ff353535ff363636ff363636ff3636365535353511363636ff363636ff363636ff343434ff4a4a4aff636363ff555555ff454545ff3c3c3cff353535ff363636ff363636ff373737edffffff01ffffff01ffffff01ffffff013a3a3ad5363636ff363636ff363636ff363636ff353535ff3f3f3fff363636ff353535ff363636ff363636ff363636ff363636ff3636365535353511363636ff363636ff363636ff363636ff353535ff383838ff3a3a3aff373737ff353535ff363636ff363636ff363636ff373737edffffff01ffffff01ffffff01ffffff013a3a3ad5363636ff363636ff363636ff323232ff181818ff0e0e0eff171717ff282828ff373737ff363636ff363636ff363636ff3636365535353511363636ff363636ff353535ff373737ff292929ff0f0f0fff111111ff1b1b1bff2f2f2fff373737ff363636ff363636ff373737edffffff01ffffff01ffffff01ffffff013a3a3ad5363636ff363636ff1e1e1eff0b0b0bff0d0d0dff0f0f0fff171717ff161616ff191919ff2c2c2cff373737ff363636ff3636365535353511363636ff373737ff2f2f2fff141414ff0b0b0bff0d0d0dff131313ff171717ff151515ff1f1f1fff333333ff363636ff373737edffffff01ffffff01ffffff01ffffff013b3b3bd5252525ff0d0d0dff0c0c0cff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff151515ff1c1c1cff313131ff3535355734343411333333ff1a1a1aff0b0b0bff0d0d0dff0d0d0dff0d0d0dff131313ff171717ff171717ff171717ff161616ff242424ff373737efffffff01ffffff01ffffff01ffffff012020205d0b0b0be50b0b0bff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff131313ff161616b73333331f3b3b3b05111111970a0a0afb0d0d0dff0d0d0dff0d0d0dff0d0d0dff131313ff171717ff171717ff171717ff161616ff141414f51c1c1c7fffffff01ffffff01ffffff01ffffff01ffffff014d4d4d0b1212127f0a0a0af50d0d0dff0d0d0dff0f0f0fff171717ff171717ff151515ff151515d522222249ffffff017373731b51515121ffffff011d1d1d2b101010b50a0a0aff0d0d0dff0d0d0dff131313ff171717ff171717ff131313ff181818a12e2e2e1dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012c2c2c1b0f0f0fa10a0a0afd0f0f0fff161616ff141414e91b1b1b69656565057878780b6363637b626262f3464646f7454545896969690fffffff011c1c1c470c0c0cd30b0b0bff131313ff141414ff151515c32a2a2a37ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff011d1d1d35111111bd1a1a1a8d2f2f2f11ffffff0166666659616161e1626262ff646464ff474747ff454545ff444444e9494949677b7b7b054040400517171769131313cd24242455ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0169696939626262c7616161ff636363ff636363ff646464ff474747ff464646ff464646ff444444ff454545d14e4e4e45ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01424242615e5e5eff636363ff636363ff636363ff636363ff646464ff474747ff464646ff464646ff464646ff464646ff434343ff3f3f3f77ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0136363679343434ff494949ff636363ff636363ff636363ff646464ff474747ff464646ff464646ff474747ff3d3d3dff353535ff3a3a3a8dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0136363679363636ff353535ff363636ff505050ff646464ff636363ff474747ff484848ff2f2f2fff1c1c1cff323232ff363636ff3a3a3a8dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0136363679363636ff363636ff363636ff353535ff3a3a3aff5a5a5aff393939ff0f0f0fff040404ff111111ff151515ff232323ff3535358fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0136363679363636ff363636ff363636ff363636ff323232ff171717ff2a2a2aff0c0c0cff030303ff111111ff141414fb171717992e2e2e17a3a3a305ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0136363679363636ff363636ff363636ff1f1f1fff0b0b0bff0d0d0dff363636ff383838ff242424ff121212bf2a2a2a2dffffff01ffffff018484842b636363bf6d6d6d2fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0136363679373737ff252525ff0d0d0dff0c0c0cff0d0d0dff0d0d0dff373737ff363636ff353535ff39393949ffffff01ffffff01ffffff0186868629646464ff656565fb6464649b55555505ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012e2e2e650e0e0eff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0c0c0cff353535ff363636ff353535ff37373749ffffff01ffffff01ffffff0185858529656565ff525252ff353535ff4b4b4b0fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff011c1c1c430d0d0dcf0b0b0bff0d0d0dff0d0d0dff0d0d0dff171717ff282828ff363636ff37373749ffffff01ffffff01ffffff0144444459363636ff353535ff353535ff4e4e4e0fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0162626203161616630b0b0be70c0c0cff0d0d0dff171717ff161616ff171717ed3737372fffffff013e3e3e2b303030b72a2a2aff151515ff262626ff363636ff4b4b4b0fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013636360d101010850a0a0af7141414f91717178f45454511ffffff014c4c4c252c2c2cdb303030ff2d2d2dff151515ff131313ff1b1b1bad5a5a5a07ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012b2b2b2121212127ffffff01ffffff01ffffff01ffffff0161616109313131752b2b2bf1131313cd26262641ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014e4e4e1359595903ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000028000000300000006000000001002000000000008025000000000000000000000000000000000000ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0173737357545454997c7c7c11ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0176767663515151916c6c6c0dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017676762d636363bb636363ff4d4d4dff434343eb4f4f4f6d7f7f7f05ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0176767635616161c3626262ff494949ff424242e94f4f4f6392929203ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017e7e7e19626262955f5f5ffd626262ff666666ff4f4f4fff464646ff424242ff434343d75a5a5a49ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017777771d6464649f5f5f5fff636363ff656565ff4b4b4bff464646ff424242ff444444d158585841ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff018585850966666677606060ef626262ff636363ff636363ff666666ff4f4f4fff464646ff464646ff464646ff414141ff464646b75d5d5d2dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff018989890d6868687f5f5f5ff5626262ff636363ff636363ff656565ff4b4b4bff464646ff464646ff464646ff404040ff484848b160606027ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff016a6a6a55626262df606060ff636363ff636363ff636363ff636363ff666666ff4f4f4fff464646ff464646ff464646ff464646ff454545ff424242fd484848956a6a6a17ffffff01ffffff01ffffff01ffffff01ffffff016969695f606060e3606060ff636363ff636363ff636363ff636363ff656565ff4b4b4bff464646ff464646ff464646ff464646ff454545ff414141f94a4a4a8d65656513ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff016e6e6e3b656565c15f5f5fff636363ff636363ff636363ff636363ff636363ff636363ff666666ff4f4f4fff464646ff464646ff464646ff464646ff464646ff464646ff444444ff424242ed52525277ffffff01ffffff016c6c6c37676767c95f5f5fff636363ff636363ff636363ff636363ff636363ff636363ff656565ff4b4b4bff464646ff464646ff464646ff464646ff464646ff464646ff434343ff444444e94d4d4d6dffffff01ffffff01ffffff01ffffff01ffffff01ffffff013c3c3cc5454545ff646464ff646464ff636363ff636363ff636363ff636363ff636363ff666666ff4f4f4fff464646ff464646ff464646ff464646ff464646ff464646ff474747ff424242ff333333fb34343409ffffff0131313199494949ff656565ff646464ff636363ff636363ff636363ff636363ff636363ff656565ff4b4b4bff464646ff464646ff464646ff464646ff464646ff464646ff474747ff414141ff373737ebffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf333333ff343434ff4f4f4fff666666ff636363ff636363ff636363ff636363ff666666ff4f4f4fff464646ff464646ff464646ff464646ff474747ff444444ff383838ff343434ff363636f737373707ffffff0135353597343434ff343434ff525252ff666666ff636363ff636363ff636363ff636363ff656565ff4b4b4bff464646ff464646ff464646ff464646ff474747ff444444ff383838ff343434ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff333333ff383838ff585858ff676767ff636363ff636363ff666666ff4f4f4fff464646ff464646ff474747ff464646ff3b3b3bff343434ff363636ff363636ff363636f737373707ffffff0135353597363636ff363636ff333333ff383838ff5a5a5aff666666ff636363ff636363ff656565ff4b4b4bff464646ff464646ff474747ff454545ff3a3a3aff343434ff363636ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff363636ff363636ff323232ff3d3d3dff5d5d5dff666666ff666666ff4f4f4fff464646ff474747ff3e3e3eff353535ff353535ff363636ff363636ff363636ff363636f737373707ffffff0135353597363636ff363636ff363636ff363636ff313131ff3f3f3fff5f5f5fff666666ff656565ff4b4b4bff464646ff474747ff3d3d3dff353535ff353535ff363636ff363636ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff363636ff363636ff363636ff353535ff323232ff444444ff676767ff525252ff404040ff363636ff353535ff363636ff363636ff363636ff363636ff363636ff363636f737373707ffffff0135353597363636ff363636ff363636ff363636ff363636ff353535ff323232ff464646ff676767ff4e4e4eff404040ff363636ff353535ff363636ff363636ff363636ff363636ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff363636ff363636ff363636ff363636ff353535ff383838ff2d2d2dff2b2b2bff373737ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636f737373707ffffff0135353597363636ff363636ff363636ff363636ff363636ff363636ff363636ff383838ff2c2c2cff2a2a2aff373737ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff363636ff363636ff353535ff383838ff343434ff171717ff090909ff151515ff171717ff2d2d2dff383838ff363636ff363636ff363636ff363636ff363636ff363636f737373707ffffff0135353597363636ff363636ff363636ff363636ff353535ff383838ff333333ff151515ff090909ff151515ff181818ff2f2f2fff383838ff363636ff363636ff363636ff363636ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff363636ff373737ff373737ff1f1f1fff090909ff0c0c0cff0c0c0cff171717ff171717ff141414ff1b1b1bff323232ff383838ff363636ff363636ff363636ff363636f737373707ffffff0135353597363636ff363636ff363636ff373737ff373737ff1d1d1dff0a0a0aff0c0c0cff0c0c0cff171717ff171717ff141414ff1c1c1cff333333ff383838ff353535ff363636ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf343434ff363636ff393939ff272727ff0c0c0cff0b0b0bff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff161616ff141414ff202020ff353535ff373737ff363636ff363636f737373707ffffff0135353597363636ff363636ff383838ff252525ff0b0b0bff0b0b0bff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff161616ff141414ff222222ff363636ff373737ff363636ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040bf383838ff2d2d2dff101010ff0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff161616ff141414ff262626ff373737ff373737f737373707ffffff0136363697393939ff2b2b2bff0f0f0fff0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff161616ff151515ff272727ff383838ff393939e3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013a3a3abd131313ff090909ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff171717ff171717ff151515ff171717ff262626fb38383807ffffff012a2a2a97121212ff090909ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff171717ff171717ff151515ff161616ff2a2a2ae7ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015f5f5f0b1616167b090909ef0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff171717ff171717ff0f0f0fff181818b74040402dffffff01ffffff014646461118181883080808f30b0b0bff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff171717ff161616ff101010ff181818b141414127ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014d4d4d171212129b090909fd0c0c0cff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff111111ff141414d335353547ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013838381d131313a5060606ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff171717ff171717ff111111ff181818cd2e2e2e3dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01333333310f0f0fbb070707ff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff141414ff121212e72424246d86868603ffffff01ffffff017373732b656565b9464646c95e5e5e3bffffff01ffffff01ffffff01323232370e0e0ec3080808ff0d0d0dff0d0d0dff0c0c0cff171717ff171717ff171717ff121212ff161616e525252563ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012525254d0e0e0ed9090909ff0c0c0cff171717ff151515ff121212f91d1d1d894d4d4d13ffffff01ffffff0178787815656565935f5f5ffb646464ff484848ff404040ff454545a96a6a6a1fffffff01ffffff01ffffff011b1b1b570e0e0edf080808ff0d0d0dff171717ff151515ff0f0f0ff3212121815656560dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01636363071a1a1a710a0a0aed0f0f0fff1b1b1bad2f2f2f23ffffff01ffffff018d8d8d0566666675616161eb616161ff636363ff646464ff484848ff464646ff454545ff424242f54c4c4c856262620fffffff01ffffff014040400b21212179080808f10f0f0fff1b1b1ba15757571dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014141411740404037ffffff01ffffff01ffffff016a6a6a4d616161db606060ff636363ff636363ff636363ff646464ff484848ff464646ff464646ff464646ff434343ff434343e751515167ffffff01ffffff01ffffff014646461d30303033ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0176767631616161c35f5f5fff636363ff636363ff636363ff636363ff636363ff646464ff484848ff464646ff464646ff464646ff464646ff464646ff424242ff454545d158585841ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015252527f636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff646464ff484848ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff434343ff454545a1ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01313131b53b3b3bff5b5b5bff676767ff636363ff636363ff636363ff636363ff636363ff646464ff484848ff464646ff464646ff464646ff464646ff464646ff474747ff444444ff393939ff383838d3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff323232ff404040ff616161ff656565ff626262ff636363ff636363ff646464ff484848ff464646ff464646ff454545ff494949ff474747ff3b3b3bff343434ff353535ff3a3a3ad3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff363636ff353535ff323232ff484848ff656565ff646464ff636363ff646464ff484848ff464646ff474747ff494949ff242424ff282828ff383838ff363636ff363636ff3a3a3ad3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff363636ff363636ff363636ff343434ff343434ff515151ff666666ff656565ff484848ff4b4b4bff323232ff070707ff040404ff151515ff181818ff2f2f2fff383838ff3a3a3ad3ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff363636ff363636ff363636ff363636ff363636ff333333ff383838ff5f5f5fff3c3c3cff0f0f0fff020202ff050505ff050505ff171717ff171717ff141414ff1c1c1cff323232d7ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff363636ff363636ff363636ff363636ff353535ff383838ff343434ff161616ff2a2a2aff0c0c0cff020202ff050505ff050505ff171717ff171717ff101010ff161616bf2e2e2e35ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff363636ff363636ff363636ff373737ff383838ff1f1f1fff0a0a0aff0c0c0cff373737ff3a3a3aff262626ff060606ff040404ff121212ff151515dd30303051ffffff01ffffff01ffffff018787872d6b6b6b47ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff363636ff363636ff393939ff272727ff0d0d0dff0b0b0bff0d0d0dff0d0d0dff373737ff363636ff373737ff383838ff1c1c1cf92020207568686807ffffff01ffffff01ffffff01ffffff018686863d5f5f5fff676767af77777721ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01363636b3363636ff393939ff2e2e2eff101010ff0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff373737ff363636ff363636ff353535ff373737ebffffff01ffffff01ffffff01ffffff01ffffff01ffffff018686863d626262ff666666ff646464f76969698d9494940fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01383838b5333333ff161616ff090909ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff373737ff363636ff363636ff363636ff353535ebffffff01ffffff01ffffff01ffffff01ffffff01ffffff018686863d626262ff676767ff6b6b6bff555555ff3a3a3a93ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0125252589030303ff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff333333ff383838ff353535ff363636ff353535ebffffff01ffffff01ffffff01ffffff01ffffff01ffffff018585853d666666ff5f5f5fff3c3c3cff313131ff3a3a3a93ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012d2d2d3f0e0e0ecb080808ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff141414ff222222ff363636ff373737ff353535ebffffff01ffffff01ffffff01ffffff01ffffff01ffffff0177777741414141ff313131ff363636ff353535ff3a3a3a93ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff011e1e1e5f0a0a0ae50a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff171717ff161616ff151515ff282828ff353535f3ffffff01ffffff01ffffff01ffffff016e6e6e0b37373781242424f1191919ff333333ff383838ff343434ff3a3a3a93ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015a5a5a0d1919197f0a0a0af30b0b0bff0d0d0dff0d0d0dff171717ff171717ff161616ff0f0f0ffb24242489ffffff01ffffff01ffffff013e3e3e5d2d2d2de52e2e2eff2b2b2bff151515ff141414ff212121ff363636ff3b3b3b95ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013636361b111111a3080808ff0c0c0cff181818ff0f0f0fff171717b545454525ffffff01ffffff017f7f7f05363636c7282828ff313131ff313131ff2b2b2bff151515ff171717ff161616ff0c0c0cfb3434346bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01303030350f0f0fc7121212d337373741ffffff01ffffff01ffffff01ffffff01ffffff016b6b6b0b3a3a3a7d2c2c2cf12f2f2fff2b2b2bff151515ff101010ff171717bb4646462dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01515151193535359b242424ff131313d72828284bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014e4e4e2b59595905ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff000000000000ffff28000000400000008000000001002000000000000042000000000000000000000000000000000000ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0176767635666666914e4e4e457c7c7c09ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff018080801569696989545454696c6c6c0bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff018484840d70707061616161d5606060fb3d3d3ddf4e4e4e9172727213ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017070704d626262b35f5f5ffb464646f1454545a16a6a6a33ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017676760f67676753646464cf5e5e5eff656565ff626262ff414141ff404040ff444444e54b4b4b7b69696919ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01979797036c6c6c45676767a95d5d5dff616161ff626262ff484848ff424242ff3e3e3efd4e4e4e8958585831ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017e7e7e2b616161a75f5f5fef616161ff636363ff656565ff626262ff424242ff464646ff444444ff414141fd434343b961616153ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017777771969696981606060e7606060ff636363ff636363ff626262ff484848ff464646ff454545ff424242fd414141d95656566569696911ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01858585056e6e6e29656565995f5f5ff1616161ff636363ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff444444ff3f3f3fff484848af5353534b86868607ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01797979216a6a6a6f616161ed5e5e5eff636363ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff3e3e3eff474747d75151515762626213ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01838383036f6f6f755f5f5fd3606060ff626262ff636363ff636363ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff464646ff454545ff434343ff404040e94e4e4e8d5f5f5f1bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff018f8f8f056b6b6b45616161c95f5f5ff7616161ff636363ff636363ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff464646ff444444ff424242f1434343b16666662dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017070700f6969695f626262d35e5e5eff626262ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff404040ff444444f14d4d4d776a6a6a23ffffff01ffffff01ffffff01ffffff017b7b7b096c6c6c39636363c15f5f5ffb626262ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff434343ff414141f54a4a4aa35b5b5b2d70707007ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0171717143676767a7616161f3616161ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff444444ff414141f7474747cd54545447ffffff01ffffff015b5b5b096b6b6b99646464e1606060ff626262ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff444444ff424242ff414141d552525277ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01404040b33b3b3bff5c5c5cff656565ff646464ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff474747ff454545ff3a3a3aff313131ad34343407ffffff012e2e2e25383838ff535353ff656565ff656565ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff474747ff464646ff3b3b3bff3a3a3ae9ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9313131ff363636ff484848ff636363ff676767ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff404040ff363636ff343434ff353535a537373705ffffff0135353521333333ff333333ff434343ff5c5c5cff686868ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff484848ff414141ff393939ff313131ff3c3c3cdbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff323232ff353535ff4b4b4bff636363ff656565ff636363ff626262ff636363ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff464646ff474747ff464646ff414141ff363636ff343434ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff333333ff313131ff484848ff5e5e5eff666666ff646464ff626262ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff464646ff464646ff474747ff424242ff3a3a3aff343434ff353535ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff343434ff333333ff3d3d3dff555555ff686868ff656565ff626262ff636363ff656565ff626262ff424242ff464646ff464646ff464646ff484848ff444444ff393939ff353535ff353535ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff353535ff323232ff363636ff515151ff646464ff656565ff636363ff636363ff636363ff626262ff484848ff464646ff464646ff464646ff484848ff454545ff3d3d3dff353535ff343434ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff363636ff343434ff303030ff3f3f3fff575757ff666666ff656565ff646464ff626262ff424242ff464646ff474747ff454545ff3a3a3aff343434ff353535ff363636ff363636ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff363636ff363636ff363636ff303030ff373737ff535353ff636363ff656565ff636363ff626262ff484848ff464646ff474747ff454545ff3e3e3eff353535ff343434ff363636ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff363636ff363636ff363636ff333333ff333333ff484848ff606060ff696969ff626262ff434343ff474747ff3e3e3eff363636ff353535ff353535ff363636ff363636ff363636ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff363636ff363636ff363636ff353535ff343434ff333333ff3e3e3eff5d5d5dff686868ff626262ff484848ff474747ff424242ff373737ff353535ff353535ff363636ff363636ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff323232ff323232ff505050ff616161ff3d3d3dff373737ff343434ff353535ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff343434ff313131ff434343ff606060ff464646ff383838ff343434ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff3a3a3aff2b2b2bff1e1e1eff2d2d2dff383838ff373737ff353535ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff393939ff323232ff1c1c1cff262626ff373737ff383838ff353535ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff363636ff363636ff353535ff373737ff383838ff303030ff191919ff080808ff101010ff141414ff1a1a1aff303030ff383838ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff363636ff363636ff363636ff353535ff363636ff383838ff363636ff1d1d1dff0b0b0bff0c0c0cff141414ff181818ff292929ff373737ff373737ff363636ff363636ff363636ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff363636ff353535ff393939ff363636ff222222ff0c0c0cff0a0a0aff0c0c0cff121212ff171717ff151515ff161616ff212121ff353535ff393939ff363636ff363636ff363636ff363636ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff363636ff363636ff353535ff383838ff3a3a3aff262626ff121212ff0a0a0aff0c0c0cff0f0f0fff171717ff151515ff151515ff1e1e1eff2f2f2fff3a3a3aff363636ff363636ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff363636ff363636ff363636ff383838ff363636ff262626ff0d0d0dff090909ff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff141414ff151515ff232323ff353535ff383838ff363636ff353535ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff353535ff383838ff383838ff292929ff131313ff080808ff0c0c0cff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff151515ff131313ff202020ff313131ff383838ff363636ff363636ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9343434ff353535ff363636ff3a3a3aff2e2e2eff131313ff0a0a0aff0b0b0bff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff161616ff141414ff1a1a1aff2a2a2aff393939ff373737ff363636ff363636ff363636a537373705ffffff0135353521363636ff363636ff363636ff3a3a3aff313131ff1c1c1cff0a0a0aff0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff161616ff151515ff161616ff282828ff363636ff383838ff363636ff333333ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444a9353535ff383838ff313131ff151515ff080808ff0b0b0bff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff171717ff171717ff161616ff131313ff1b1b1bff2d2d2dff373737ff373737ff363636a537373705ffffff0134343421363636ff383838ff333333ff1e1e1eff090909ff0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff171717ff171717ff171717ff131313ff171717ff2a2a2aff363636ff353535ff3d3d3ddbffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01444444af353535ff1e1e1eff0d0d0dff0a0a0aff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff151515ff151515ff222222ff333333ff353535ad30303007ffffff0134343423373737ff282828ff0d0d0dff0a0a0aff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff151515ff141414ff1b1b1bff2e2e2eff3e3e3ee1ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013e3e3e6f0f0f0fd5040404ff0b0b0bff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff101010ff0e0e0ee72f2f2f7347474703ffffff013b3b3b13141414cd050505f70a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff121212ff0c0c0cf12a2a2aa5ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015f5f5f052020202b1a1a1aa1080808f1070707ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff141414ff0c0c0cff212121af2a2a2a496d6d6d07ffffff01ffffff01ffffff01333333231d1d1d730b0b0beb060606ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff171717ff171717ff171717ff171717ff151515ff0e0e0eff181818d72626265546464615ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014d4d4d29121212af080808ef0a0a0aff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff171717ff171717ff141414ff121212f9141414b93b3b3b4fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0138383819151515890a0a0ae5080808ff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff171717ff171717ff161616ff101010fb151515d72c2c2c614444440dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0133333311262626510f0f0fd7050505ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff121212ff171717ff171717ff171717ff171717ff171717ff101010ff141414e7242424733a3a3a19ffffff01ffffff01ffffff01878787097272725f4d4d4d736a6a6a11ffffff01ffffff01ffffff016060600524242445191919ad040404ff0a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff171717ff171717ff171717ff111111ff0e0e0efd242424873232322dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015c5c5c0d2525255f090909d7080808fb0b0b0bff0d0d0dff0c0c0cff121212ff171717ff171717ff161616ff121212ff121212df2121218965656511ffffff01ffffff01ffffff018080800d6767674b646464d1606060ff454545ff464646df4f4f4f6165656517ffffff01ffffff01ffffff01ffffff012d2d2d4b101010b5060606fb0a0a0aff0d0d0dff0d0d0dff0f0f0fff171717ff171717ff161616ff131313ff101010ef2020209d4242422dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012c2c2c2d1f1f1f83080808fb080808ff0d0d0dff121212ff171717ff141414ff0f0f0ff91e1e1eb12c2c2c354d4d4d09ffffff01ffffff01ffffff0178787825646464a75f5f5feb616161ff656565ff4a4a4aff414141ff424242f3414141bd69696937ffffff01ffffff01ffffff01ffffff0142424219171717710d0d0de3060606ff0c0c0cff0f0f0fff171717ff151515ff0d0d0dff171717c3292929575656560dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013737372d1212129d080808ef0d0d0dff121212f5191919bf2e2e2e3d70707003ffffff01ffffff018c8c8c037676762564646497606060ed606060ff636363ff636363ff656565ff4a4a4aff444444ff464646ff444444ff404040f74a4a4aad5555553162626207ffffff01ffffff01ffffff014040401125252589090909dd0a0a0aff121212ff141414c738383869ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015b5b5b0b1f1f1f591d1d1daf292929673f3f3f19ffffff01ffffff01ffffff01ffffff016d6d6d715f5f5fcd606060ff626262ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff454545ff434343ff414141db4f4f4f857b7b7b11ffffff01ffffff01ffffff0153535307222222331d1d1da91b1b1b8d4141412365656503ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff017c7c7c0f6868685d636363cb5e5e5eff626262ff636363ff636363ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff464646ff464646ff464646ff404040ff454545e14c4c4c6b69696917ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0177777733626262a3606060f3616161ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff464646ff464646ff464646ff464646ff444444ff424242f9454545b55d5d5d49ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014b4b4b0f5e5e5e85626262ff626262ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff444444ff414141ff454545a16464641dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0132323225333333cf4e4e4eff646464ff666666ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff474747ff404040ff303030e35757573bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd313131ff363636ff515151ff636363ff656565ff636363ff636363ff636363ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff464646ff464646ff464646ff464646ff464646ff414141ff373737ff343434ff323232e159595939ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff343434ff333333ff3c3c3cff5b5b5bff686868ff636363ff626262ff636363ff636363ff636363ff656565ff4a4a4aff444444ff464646ff464646ff454545ff464646ff4c4c4cff454545ff393939ff353535ff353535ff353535ff323232e159595939ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff353535ff313131ff3f3f3fff5d5d5dff666666ff646464ff626262ff636363ff656565ff4a4a4aff444444ff454545ff474747ff4a4a4aff404040ff212121ff2f2f2fff373737ff373737ff353535ff363636ff323232e159595939ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff363636ff353535ff333333ff363636ff484848ff646464ff676767ff626262ff656565ff4a4a4aff444444ff4b4b4bff4a4a4aff262626ff0b0b0bff090909ff171717ff252525ff353535ff393939ff363636ff323232e159595939ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff363636ff363636ff363636ff363636ff323232ff363636ff4c4c4cff646464ff676767ff4d4d4dff484848ff2c2c2cff0b0b0bff020202ff040404ff0b0b0bff171717ff141414ff161616ff282828ff353535ff343434e359595939ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff363636ff363636ff363636ff363636ff363636ff343434ff323232ff3f3f3fff5f5f5fff3a3a3aff161616ff030303ff030303ff050505ff040404ff0b0b0bff171717ff171717ff161616ff151515ff1a1a1aff242424e55555553bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff363636ff363636ff363636ff363636ff353535ff363636ff383838ff2e2e2eff191919ff262626ff111111ff030303ff030303ff050505ff040404ff0b0b0bff171717ff171717ff151515ff111111f9121212cd272727557d7d7d09ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff363636ff363636ff363636ff363636ff383838ff373737ff242424ff0b0b0bff0a0a0aff393939ff393939ff222222ff080808ff020202ff030303ff0b0b0bff181818ff0f0f0fff151515f32424247935353525ffffff01ffffff01ffffff01a3a3a30fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff363636ff363636ff363636ff383838ff373737ff272727ff0c0c0cff090909ff0c0c0cff0e0e0eff373737ff363636ff3a3a3aff393939ff1e1e1eff080808ff080808ff0f0f0feb232323914040401dffffff01ffffff01ffffff01ffffff01ffffff018282825d626262c36d6d6d4d8d8d8d09ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff363636ff353535ff363636ff3a3a3aff2f2f2fff131313ff0b0b0bff0b0b0bff0d0d0dff0c0c0cff0e0e0eff373737ff363636ff353535ff363636ff393939ff303030ff1c1c1cc92626264d68686807ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01868686515e5e5eff646464e9696969957878781fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723363636cd363636ff373737ff383838ff313131ff161616ff090909ff0b0b0bff0d0d0dff0d0d0dff0d0d0dff0c0c0cff0e0e0eff373737ff363636ff363636ff363636ff353535ff353535ff3c3c3c8fffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0186868651616161ff676767ff646464ff656565f16a6a6a7d7f7f7f25ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0137373723353535cd393939ff373737ff1f1f1fff0d0d0dff0a0a0aff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff0e0e0eff373737ff363636ff363636ff363636ff363636ff353535ff37373791ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0186868651616161ff676767ff666666ff676767ff686868f9555555cd55555511ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0134343425323232cf212121ff0e0e0eff090909ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff0e0e0eff383838ff363636ff363636ff363636ff363636ff353535ff37373791ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0186868651616161ff686868ff696969ff5f5f5fff3d3d3dff303030ff4848481dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01474747132323238f020202ff080808ff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff0c0c0cff2e2e2eff393939ff363636ff353535ff363636ff353535ff37373791ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0185858551666666ff676767ff494949ff353535ff323232ff353535ff4e4e4e1bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0130303045101010af080808f70a0a0aff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff0d0d0dff131313ff1c1c1cff303030ff373737ff363636ff353535ff37373791ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0181818151494949ff363636ff313131ff363636ff353535ff363636ff4e4e4e1bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff0141414113191919690f0f0fdb060606ff0c0c0cff0d0d0dff0d0d0dff0d0d0dff0d0d0dff0c0c0cff0d0d0dff171717ff151515ff161616ff222222ff363636ff383838ff37373791ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff014d4d4d53272727c1242424ff373737ff373737ff353535ff353535ff363636ff4e4e4e1bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01626262091c1c1c830b0b0bd7090909ff0c0c0cff0d0d0dff0d0d0dff0c0c0cff0d0d0dff171717ff171717ff171717ff141414ff151515ff202020ff35353595ffffff01ffffff01ffffff01ffffff017474740540404049343434af2a2a2aff262626ff101010ff191919ff2e2e2eff373737ff363636ff363636ff4e4e4e1bffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015a5a5a073636362d141414a7080808f5080808ff0d0d0dff0c0c0cff0d0d0dff171717ff171717ff171717ff151515ff0e0e0efb1b1b1bbb3d3d3d29ffffff01ffffff01ffffff0151515119393939892a2a2ae92d2d2dff323232ff282828ff141414ff151515ff151515ff1f1f1fff343434ff393939ff4949491dffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff013636362f111111b5070707f30a0a0aff0d0d0dff171717ff141414ff111111f5111111c74343433d70707005ffffff01ffffff017c7c7c034e4e4e632a2a2af7292929ff323232ff313131ff323232ff282828ff141414ff171717ff171717ff151515ff0e0e0efd222222e153535315ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff012d2d2d151f1f1f590e0e0edb040404ff0f0f0fff171717e7262626673f3f3f1dffffff01ffffff01ffffff01ffffff01ffffff01444444293535358b2d2d2deb2b2b2bff313131ff323232ff282828ff141414ff171717ff121212ff0d0d0dff2222229d2626263dbebebe03ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01505050112626266f1d1d1d7f36363617ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01616161213333339d2c2c2ce92f2f2fff282828ff111111ff111111f7191919ab3c3c3c41ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015151510b3b3b3b43383838c51f1f1fff141414d71e1e1e654f4f4f13ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff015858580b4d4d4d4159595909ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff01ffffff010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000` diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 198ca22ce0..2a163dd39c 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -106,13 +106,18 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC } // AddEntry stores the given data and adds the resulting key to the manifest -func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (storage.Address, error) { - key, _, err := m.api.Store(ctx, data, e.Size, m.trie.encrypted) - if err != nil { - return nil, err - } +func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (key storage.Address, err error) { entry := newManifestTrieEntry(e, nil) - entry.Hash = key.Hex() + if data != nil { + key, _, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) + if err != nil { + return nil, err + } + entry.Hash = key.Hex() + } + if entry.Hash == "" { + return key, errors.New("missing entry hash") + } m.trie.addEntry(entry, m.quitC) return key, nil } @@ -159,7 +164,7 @@ func (m *ManifestWalker) Walk(walkFn WalkFn) error { } func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error { - for _, entry := range trie.entries { + for _, entry := range &trie.entries { if entry == nil { continue } @@ -308,7 +313,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { } func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { - for _, e := range mt.entries { + for _, e := range &mt.entries { if e != nil { cnt++ entry = e @@ -362,7 +367,7 @@ func (mt *manifestTrie) recalcAndStore() error { buffer.WriteString(`{"entries":[`) list := &Manifest{} - for _, entry := range mt.entries { + for _, entry := range &mt.entries { if entry != nil { if entry.Hash == "" { // TODO: paralellize err := entry.subtrie.recalcAndStore() diff --git a/swarm/bmt/bmt.go b/swarm/bmt/bmt.go index 1a141047ae..97e0e141ed 100644 --- a/swarm/bmt/bmt.go +++ b/swarm/bmt/bmt.go @@ -318,7 +318,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) { // with every full segment calls writeSection in a go routine func (h *Hasher) Write(b []byte) (int, error) { l := len(b) - if l == 0 { + if l == 0 || l > 4096 { return 0, nil } t := h.getTree() diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 49ae5a15bf..7e7fee8efe 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -94,12 +94,14 @@ type BzzConfig struct { UnderlayAddr []byte // node's underlay address HiveParams *HiveParams NetworkID uint64 + LightNode bool } // Bzz is the swarm protocol bundle type Bzz struct { *Hive NetworkID uint64 + LightNode bool localAddr *BzzAddr mtx sync.Mutex handshakes map[discover.NodeID]*HandshakeMsg @@ -116,6 +118,7 @@ func NewBzz(config *BzzConfig, kad Overlay, store state.Store, streamerSpec *pro return &Bzz{ Hive: NewHive(config.HiveParams, kad, store), NetworkID: config.NetworkID, + LightNode: config.LightNode, localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr}, handshakes: make(map[discover.NodeID]*HandshakeMsg), streamerRun: streamerRun, @@ -209,7 +212,11 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* localAddr: b.localAddr, BzzAddr: handshake.peerAddr, lastActive: time.Now(), + LightNode: handshake.LightNode, } + + log.Debug("peer created", "addr", handshake.peerAddr.String()) + return run(peer) } } @@ -228,6 +235,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error return err } handshake.peerAddr = rsh.(*HandshakeMsg).Addr + handshake.LightNode = rsh.(*HandshakeMsg).LightNode return nil } @@ -263,6 +271,7 @@ type BzzPeer struct { localAddr *BzzAddr // local Peers address *BzzAddr // remote address -> implements Addr interface = protocols.Peer lastActive time.Time // time is updated whenever mutexes are releasing + LightNode bool } func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer { @@ -294,6 +303,7 @@ type HandshakeMsg struct { Version uint64 NetworkID uint64 Addr *BzzAddr + LightNode bool // peerAddr is the address received in the peer handshake peerAddr *BzzAddr @@ -305,7 +315,7 @@ type HandshakeMsg struct { // String pretty prints the handshake func (bh *HandshakeMsg) String() string { - return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v", bh.Version, bh.NetworkID, bh.Addr) + return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, LightNode: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.LightNode, bh.peerAddr) } // Perform initiates the handshake and validates the remote handshake message @@ -338,6 +348,7 @@ func (b *Bzz) GetHandshake(peerID discover.NodeID) (*HandshakeMsg, bool) { Version: uint64(BzzSpec.Version), NetworkID: b.NetworkID, Addr: b.localAddr, + LightNode: b.LightNode, init: make(chan bool, 1), done: make(chan struct{}), } diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 63faf31088..b74b72c68b 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -30,6 +30,11 @@ import ( p2ptest "github.com/ethereum/go-ethereum/p2p/testing" ) +const ( + TestProtocolVersion = 5 + TestProtocolNetworkID = 3 +) + var ( loglevel = flag.Int("loglevel", 2, "verbosity of logs") ) @@ -127,23 +132,30 @@ type bzzTester struct { *p2ptest.ProtocolTester addr *BzzAddr cs map[string]chan bool + bzz *Bzz } -func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr) *bzzTester { +func newBzz(addr *BzzAddr, lightNode bool) *Bzz { config := &BzzConfig{ OverlayAddr: addr.Over(), UnderlayAddr: addr.Under(), HiveParams: NewHiveParams(), NetworkID: DefaultNetworkID, + LightNode: lightNode, } kad := NewKademlia(addr.OAddr, NewKadParams()) bzz := NewBzz(config, kad, nil, nil, nil) + return bzz +} - s := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, bzz.runBzz) +func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *bzzTester { + bzz := newBzz(addr, lightNode) + pt := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), n, bzz.runBzz) return &bzzTester{ addr: addr, - ProtocolTester: s, + ProtocolTester: pt, + bzz: bzz, } } @@ -184,22 +196,24 @@ func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptes return nil } -func correctBzzHandshake(addr *BzzAddr) *HandshakeMsg { +func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg { return &HandshakeMsg{ - Version: 5, - NetworkID: DefaultNetworkID, + Version: TestProtocolVersion, + NetworkID: TestProtocolNetworkID, Addr: addr, + LightNode: lightNode, } } func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { + lightNode := false addr := RandomAddr() - s := newBzzHandshakeTester(t, 1, addr) + s := newBzzHandshakeTester(t, 1, addr, lightNode) id := s.IDs[0] err := s.testHandshake( - correctBzzHandshake(addr), - &HandshakeMsg{Version: 5, NetworkID: 321, Addr: NewAddrFromNodeID(id)}, + correctBzzHandshake(addr, lightNode), + &HandshakeMsg{Version: TestProtocolVersion, NetworkID: 321, Addr: NewAddrFromNodeID(id)}, &p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): network id mismatch 321 (!= 3)")}, ) @@ -209,14 +223,15 @@ func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { } func TestBzzHandshakeVersionMismatch(t *testing.T) { + lightNode := false addr := RandomAddr() - s := newBzzHandshakeTester(t, 1, addr) + s := newBzzHandshakeTester(t, 1, addr, lightNode) id := s.IDs[0] err := s.testHandshake( - correctBzzHandshake(addr), - &HandshakeMsg{Version: 0, NetworkID: 3, Addr: NewAddrFromNodeID(id)}, - &p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= 5)")}, + correctBzzHandshake(addr, lightNode), + &HandshakeMsg{Version: 0, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)}, + &p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= %d)", TestProtocolVersion)}, ) if err != nil { @@ -225,16 +240,49 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) { } func TestBzzHandshakeSuccess(t *testing.T) { + lightNode := false addr := RandomAddr() - s := newBzzHandshakeTester(t, 1, addr) + s := newBzzHandshakeTester(t, 1, addr, lightNode) id := s.IDs[0] err := s.testHandshake( - correctBzzHandshake(addr), - &HandshakeMsg{Version: 5, NetworkID: 3, Addr: NewAddrFromNodeID(id)}, + correctBzzHandshake(addr, lightNode), + &HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)}, ) if err != nil { t.Fatal(err) } } + +func TestBzzHandshakeLightNode(t *testing.T) { + var lightNodeTests = []struct { + name string + lightNode bool + }{ + {"on", true}, + {"off", false}, + } + + for _, test := range lightNodeTests { + t.Run(test.name, func(t *testing.T) { + randomAddr := RandomAddr() + pt := newBzzHandshakeTester(t, 1, randomAddr, false) + id := pt.IDs[0] + addr := NewAddrFromNodeID(id) + + err := pt.testHandshake( + correctBzzHandshake(randomAddr, false), + &HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: addr, LightNode: test.lightNode}, + ) + + if err != nil { + t.Fatal(err) + } + + if pt.bzz.handshakes[id].LightNode != test.lightNode { + t.Fatalf("peer LightNode flag is %v, should be %v", pt.bzz.handshakes[id].LightNode, test.lightNode) + } + }) + } +} diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index 2241dfca21..74f9d98ee9 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -62,6 +62,8 @@ type Simulation struct { // where all "global" state related to the service should be kept. // All cleanups needed for constructed service and any other constructed // objects should ne provided in a single returned cleanup function. +// Returned cleanup function will be called by Close function +// after network shutdown. type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) // New creates a new Simulation instance with new @@ -161,6 +163,18 @@ var maxParallelCleanups = 10 // simulation. func (s *Simulation) Close() { close(s.done) + + // Close all connections before calling the Network Shutdown. + // It is possible that p2p.Server.Stop will block if there are + // existing connections. + for _, c := range s.Net.Conns { + if c.Up { + s.Net.Disconnect(c.One, c.Other) + } + } + s.shutdownWG.Wait() + s.Net.Shutdown() + sem := make(chan struct{}, maxParallelCleanups) s.mu.RLock() cleanupFuncs := make([]func(), len(s.cleanupFuncs)) @@ -170,16 +184,19 @@ func (s *Simulation) Close() { } } s.mu.RUnlock() + var cleanupWG sync.WaitGroup for _, cleanup := range cleanupFuncs { - s.shutdownWG.Add(1) + cleanupWG.Add(1) sem <- struct{}{} go func(cleanup func()) { - defer s.shutdownWG.Done() + defer cleanupWG.Done() defer func() { <-sem }() cleanup() }(cleanup) } + cleanupWG.Wait() + if s.httpSrv != nil { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -189,8 +206,6 @@ func (s *Simulation) Close() { } close(s.runC) } - s.shutdownWG.Wait() - s.Net.Shutdown() } // Done returns a channel that is closed when the simulation diff --git a/swarm/network/simulation/simulation_test.go b/swarm/network/simulation/simulation_test.go index 803e0499a2..8576732c9f 100644 --- a/swarm/network/simulation/simulation_test.go +++ b/swarm/network/simulation/simulation_test.go @@ -68,7 +68,7 @@ func TestRun(t *testing.T) { defer cancel() r := sim.Run(ctx, func(ctx context.Context, sim *Simulation) error { - time.Sleep(100 * time.Millisecond) + time.Sleep(time.Second) return nil }) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index ae007e5b0a..972cc859a3 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -393,6 +393,11 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return err } + log.Debug("Waiting for kademlia") + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + return err + } + //each of the nodes (except pivot node) subscribes to the stream of the next node for j, node := range nodeIDs[0 : nodes-1] { sid := nodeIDs[j+1] @@ -424,11 +429,6 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } }() - log.Debug("Waiting for kademlia") - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { - return err - } - log.Debug("Watching for disconnections") disconnections := sim.PeerEvents( context.Background(), diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 2dfc5898fe..6acab50af4 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -246,6 +246,8 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { if err != nil { log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) localSuccess = false + // Do not get crazy with logging the warn message + time.Sleep(500 * time.Millisecond) } else { log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) } @@ -426,6 +428,8 @@ func testSyncingViaDirectSubscribe(chunkCount int, nodeCount int) error { if err != nil { log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) localSuccess = false + // Do not get crazy with logging the warn message + time.Sleep(500 * time.Millisecond) } else { log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) } diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go new file mode 100644 index 0000000000..8619f6e19c --- /dev/null +++ b/swarm/sctx/sctx.go @@ -0,0 +1,7 @@ +package sctx + +type ContextKey int + +const ( + HTTPRequestIDKey ContextKey = iota +) diff --git a/swarm/storage/mru/handler.go b/swarm/storage/mru/handler.go index 188b986b85..32f43d5029 100644 --- a/swarm/storage/mru/handler.go +++ b/swarm/storage/mru/handler.go @@ -469,7 +469,7 @@ func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAd log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.SData, "multihash", r.multihash) // update our resources map entry if the new update is older than the one we have, if we have it. - if rsrc != nil && r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version) { + if rsrc != nil && (r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version)) { rsrc.period = r.period rsrc.version = r.version rsrc.data = make([]byte, len(r.data)) diff --git a/swarm/swarm.go b/swarm/swarm.go index db7d2dfedc..c380a376f6 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -143,6 +143,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e OverlayAddr: addr.OAddr, UnderlayAddr: addr.UAddr, HiveParams: config.HiveParams, + LightNode: config.LightNodeEnabled, } stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db")) diff --git a/swarm/version/version.go b/swarm/version/version.go index 5e8c78823f..bb9935e24a 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 1 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/tests/init.go b/tests/init.go index af65f92069..f0a4943c13 100644 --- a/tests/init.go +++ b/tests/init.go @@ -53,6 +53,16 @@ var Forks = map[string]*params.ChainConfig{ DAOForkBlock: big.NewInt(0), ByzantiumBlock: big.NewInt(0), }, + "Constantinople": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + DAOForkBlock: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + }, "FrontierToHomesteadAt5": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(5), diff --git a/trie/database.go b/trie/database.go index 8675b9f0a0..d0691b637e 100644 --- a/trie/database.go +++ b/trie/database.go @@ -51,7 +51,7 @@ const secureKeyLength = 11 + 32 // DatabaseReader wraps the Get and Has method of a backing store for the trie. type DatabaseReader interface { - // Get retrieves the value associated with key form the database. + // Get retrieves the value associated with key from the database. Get(key []byte) (value []byte, err error) // Has retrieves whether a key is present in the database. @@ -431,6 +431,11 @@ func (db *Database) reference(child common.Hash, parent common.Hash) { // Dereference removes an existing reference from a root node. func (db *Database) Dereference(root common.Hash) { + // Sanity check to ensure that the meta-root is not removed + if root == (common.Hash{}) { + log.Error("Attempted to dereference the trie cache meta root") + return + } db.lock.Lock() defer db.lock.Unlock() diff --git a/trie/node.go b/trie/node.go index a06f1b3898..1fafb7a538 100644 --- a/trie/node.go +++ b/trie/node.go @@ -55,7 +55,7 @@ var nilValueNode = valueNode(nil) func (n *fullNode) EncodeRLP(w io.Writer) error { var nodes [17]node - for i, child := range n.Children { + for i, child := range &n.Children { if child != nil { nodes[i] = child } else { @@ -98,7 +98,7 @@ func (n valueNode) String() string { return n.fstring("") } func (n *fullNode) fstring(ind string) string { resp := fmt.Sprintf("[\n%s ", ind) - for i, node := range n.Children { + for i, node := range &n.Children { if node == nil { resp += fmt.Sprintf("%s: ", indices[i]) } else { diff --git a/trie/trie.go b/trie/trie.go index 4284e30ad4..e920ccd23f 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -356,7 +356,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // value that is left in n or -2 if n contains at least two // values. pos := -1 - for i, cld := range n.Children { + for i, cld := range &n.Children { if cld != nil { if pos == -1 { pos = i diff --git a/whisper/whisperv6/doc.go b/whisper/whisperv6/doc.go index 066a9766d4..529bf3d2de 100644 --- a/whisper/whisperv6/doc.go +++ b/whisper/whisperv6/doc.go @@ -33,7 +33,6 @@ particularly the notion of singular endpoints. package whisperv6 import ( - "fmt" "time" ) @@ -79,12 +78,6 @@ const ( DefaultSyncAllowance = 10 // seconds ) -type unknownVersionError uint64 - -func (e unknownVersionError) Error() string { - return fmt.Sprintf("invalid envelope version %d", uint64(e)) -} - // MailServer represents a mail server, capable of // archiving the old messages for subsequent delivery // to the peers. Any implementation must ensure that both