Merge pull request #20 from ethereum/master

Pull latest changes from go-ethereum

cmd/geth/main.go

@@ -199,10 +199,15 @@ func init() {
 
 	app.Before = func(ctx *cli.Context) error {
 		runtime.GOMAXPROCS(runtime.NumCPU())
-		if err := debug.Setup(ctx); err != nil {
+
+		logdir := ""
+		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
+			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")
+		}
+		if err := debug.Setup(ctx, logdir); err != nil {
 			return err
 		}
-		// Cap the cache allowance and tune the garbage colelctor
+		// Cap the cache allowance and tune the garbage collector
 		var mem gosigar.Mem
 		if err := mem.Get(); err == nil {
 			allowance := int(mem.Total / 1024 / 1024 / 3)

cmd/geth/usage.go

@@ -83,7 +83,8 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.LightKDFFlag,
 		},
 	},
-	{Name: "DEVELOPER CHAIN",
+	{
+		Name: "DEVELOPER CHAIN",
 		Flags: []cli.Flag{
 			utils.DeveloperFlag,
 			utils.DeveloperPeriodFlag,

cmd/swarm/fs_test.go

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
+// +build linux darwin freebsd
+
 package main
 
 import (

cmd/swarm/hash.go

@@ -18,6 +18,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -39,7 +40,7 @@ func hash(ctx *cli.Context) {
 
 	stat, _ := f.Stat()
 	fileStore := storage.NewFileStore(storage.NewMapChunkStore(), storage.NewFileStoreParams())
-	addr, _, err := fileStore.Store(f, stat.Size(), false)
+	addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false)
 	if err != nil {
 		utils.Fatalf("%v\n", err)
 	} else {

cmd/swarm/main.go

@@ -143,7 +143,7 @@ var (
 	}
 	SwarmWantManifestFlag = cli.BoolTFlag{
 		Name:  "manifest",
-		Usage: "Automatic manifest upload",
+		Usage: "Automatic manifest upload (default true)",
 	}
 	SwarmUploadDefaultPath = cli.StringFlag{
 		Name:  "defaultpath",
@@ -155,7 +155,7 @@ var (
 	}
 	SwarmUploadMimeType = cli.StringFlag{
 		Name:  "mime",
-		Usage: "force mime type",
+		Usage: "Manually specify MIME type",
 	}
 	SwarmEncryptedFlag = cli.BoolFlag{
 		Name:  "encrypt",
@@ -432,7 +432,7 @@ pv(1) tool to get a progress bar:
 	app.Flags = append(app.Flags, swarmmetrics.Flags...)
 	app.Before = func(ctx *cli.Context) error {
 		runtime.GOMAXPROCS(runtime.NumCPU())
-		if err := debug.Setup(ctx); err != nil {
+		if err := debug.Setup(ctx, ""); err != nil {
 			return err
 		}
 		swarmmetrics.Setup(ctx)

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -37,8 +37,14 @@ import (
 )
 
 func generateEndpoints(scheme string, cluster string, from int, to int) {
+	if cluster == "prod" {
+		cluster = ""
+	} else {
+		cluster = cluster + "."
+	}
+
 	for port := from; port <= to; port++ {
-		endpoints = append(endpoints, fmt.Sprintf("%s://%v.%s.swarm-gateways.net", scheme, port, cluster))
+		endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
 	}
 
 	if includeLocalhost {

cmd/utils/flags.go

@@ -193,7 +193,7 @@ var (
 	}
 	// Dashboard settings
 	DashboardEnabledFlag = cli.BoolFlag{
-		Name:  "dashboard",
+		Name:  metrics.DashboardEnabledFlag,
 		Usage: "Enable the dashboard",
 	}
 	DashboardAddrFlag = cli.StringFlag{
@@ -998,7 +998,7 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) {
 	}
 }
 
-// checkExclusive verifies that only a single isntance of the provided flags was
+// checkExclusive verifies that only a single instance of the provided flags was
 // set by the user. Each flag might optionally be followed by a string type to
 // specialize it further.
 func checkExclusive(ctx *cli.Context, args ...interface{}) {
@@ -1185,7 +1185,7 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 // RegisterDashboardService adds a dashboard to the stack.
 func RegisterDashboardService(stack *node.Node, cfg *dashboard.Config, commit string) {
 	stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		return dashboard.New(cfg, commit)
+		return dashboard.New(cfg, commit, ctx.ResolvePath("logs")), nil
 	})
 }
 

consensus/clique/api.go

@@ -75,7 +75,7 @@ func (api *API) GetSigners(number *rpc.BlockNumber) ([]common.Address, error) {
 	return snap.signers(), nil
 }
 
-// GetSignersAtHash retrieves the state snapshot at a given block.
+// GetSignersAtHash retrieves the list of authorized signers at the specified block.
 func (api *API) GetSignersAtHash(hash common.Hash) ([]common.Address, error) {
 	header := api.chain.GetHeaderByHash(hash)
 	if header == nil {

consensus/ethash/consensus.go

@@ -357,7 +357,7 @@ func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int {
 	}
 	// calculate a fake block number for the ice-age delay:
 	// https://github.com/ethereum/EIPs/pull/669
-	//   fake_block_number = min(0, block.number - 3_000_000
+	// fake_block_number = max(0, block.number - 3_000_000)
 	fakeBlockNumber := new(big.Int)
 	if parent.Number.Cmp(big2999999) >= 0 {
 		fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, big2999999) // Note, parent is 1 less than the actual block number

core/types/transaction.go

@@ -266,9 +266,9 @@ func (s Transactions) GetRlp(i int) []byte {
 	return enc
 }
 
-// TxDifference returns a new set t which is the difference between a to b.
+// TxDifference returns a new set which is the difference between a and b.
-func TxDifference(a, b Transactions) (keep Transactions) {
+func TxDifference(a, b Transactions) Transactions {
-	keep = make(Transactions, 0, len(a))
+	keep := make(Transactions, 0, len(a))
 
 	remove := make(map[common.Hash]struct{})
 	for _, tx := range b {

dashboard/assets/common.jsx

@@ -68,4 +68,4 @@ export const styles = {
 	light: {
 		color: 'rgba(255, 255, 255, 0.54)',
 	},
-}
+};

dashboard/assets/components/Body.jsx

@@ -37,6 +37,7 @@ export type Props = {
 	active:        string,
 	content:       Content,
 	shouldUpdate:  Object,
+	send:          string => void,
 };
 
 // Body renders the body of the dashboard.
@@ -52,6 +53,7 @@ class Body extends Component<Props> {
 					active={this.props.active}
 					content={this.props.content}
 					shouldUpdate={this.props.shouldUpdate}
+					send={this.props.send}
 				/>
 			</div>
 		);

dashboard/assets/components/CustomTooltip.jsx

@@ -85,7 +85,7 @@ export type Props = {
 class CustomTooltip extends Component<Props> {
 	render() {
 		const {active, payload, tooltip} = this.props;
-		if (!active || typeof tooltip !== 'function') {
+		if (!active || typeof tooltip !== 'function' || !Array.isArray(payload) || payload.length < 1) {
 			return null;
 		}
 		return tooltip(payload[0].value);

dashboard/assets/components/Dashboard.jsx

@@ -24,6 +24,7 @@ import Header from './Header';
 import Body from './Body';
 import {MENU} from '../common';
 import type {Content} from '../types/content';
+import {inserter as logInserter} from './Logs';
 
 // deepUpdate updates an object corresponding to the given update data, which has
 // the shape of the same structure as the original object. updater also has the same
@@ -75,8 +76,11 @@ const appender = <T>(limit: number, mapper = replacer) => (update: Array<T>, pre
 	...update.map(sample => mapper(sample)),
 ].slice(-limit);
 
-// defaultContent is the initial value of the state content.
+// defaultContent returns the initial value of the state content. Needs to be a function in order to
-const defaultContent: Content = {
+// instantiate the object again, because it is used by the state, and isn't automatically cleaned
+// when a new connection is established. The state is mutated during the update in order to avoid
+// the execution of unnecessary operations (e.g. copy of the log array).
+const defaultContent: () => Content = () => ({
 	general: {
 		version: null,
 		commit:  null,
@@ -96,9 +100,13 @@ const defaultContent: Content = {
 		diskWrite:      [],
 	},
 	logs: {
-		log: [],
+		chunks:        [],
+		endTop:        false,
+		endBottom:     true,
+		topChanged:    0,
+		bottomChanged: 0,
 	},
-};
+});
 
 // updaters contains the state updater functions for each path of the state.
 //
@@ -122,9 +130,7 @@ const updaters = {
 		diskRead:       appender(200),
 		diskWrite:      appender(200),
 	},
-	logs: {
+	logs: logInserter(5),
-		log: appender(200),
-	},
 };
 
 // styles contains the constant styles of the component.
@@ -155,6 +161,7 @@ type State = {
 	sideBar:      boolean, // true if the sidebar is opened
 	content:      Content, // the visualized data
 	shouldUpdate: Object,  // labels for the components, which need to re-render based on the incoming message
+	server:       ?WebSocket,
 };
 
 // Dashboard is the main component, which renders the whole page, makes connection with the server and
@@ -165,8 +172,9 @@ class Dashboard extends Component<Props, State> {
 		this.state = {
 			active:       MENU.get('home').id,
 			sideBar:      true,
-			content:      defaultContent,
+			content:      defaultContent(),
 			shouldUpdate: {},
+			server:       null,
 		};
 	}
 
@@ -181,7 +189,7 @@ class Dashboard extends Component<Props, State> {
 		// PROD is defined by webpack.
 		const server = new WebSocket(`${((window.location.protocol === 'https:') ? 'wss://' : 'ws://')}${PROD ? window.location.host : 'localhost:8080'}/api`);
 		server.onopen = () => {
-			this.setState({content: defaultContent, shouldUpdate: {}});
+			this.setState({content: defaultContent(), shouldUpdate: {}, server});
 		};
 		server.onmessage = (event) => {
 			const msg: $Shape<Content> = JSON.parse(event.data);
@@ -192,10 +200,18 @@ class Dashboard extends Component<Props, State> {
 			this.update(msg);
 		};
 		server.onclose = () => {
+			this.setState({server: null});
 			setTimeout(this.reconnect, 3000);
 		};
 	};
 
+	// send sends a message to the server, which can be accessed only through this function for safety reasons.
+	send = (msg: string) => {
+		if (this.state.server != null) {
+			this.state.server.send(msg);
+		}
+	};
+
 	// update updates the content corresponding to the incoming message.
 	update = (msg: $Shape<Content>) => {
 		this.setState(prevState => ({
@@ -226,6 +242,7 @@ class Dashboard extends Component<Props, State> {
 					active={this.state.active}
 					content={this.state.content}
 					shouldUpdate={this.state.shouldUpdate}
+					send={this.send}
 				/>
 			</div>
 		);

dashboard/assets/components/Logs.jsx

@@ -0,0 +1,310 @@
+// @flow
+
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+
+import List, {ListItem} from 'material-ui/List';
+import type {Record, Content, LogsMessage, Logs as LogsType} from '../types/content';
+
+// requestBand says how wide is the top/bottom zone, eg. 0.1 means 10% of the container height.
+const requestBand = 0.05;
+
+// fieldPadding is a global map with maximum field value lengths seen until now
+// to allow padding log contexts in a bit smarter way.
+const fieldPadding = new Map();
+
+// createChunk creates an HTML formatted object, which displays the given array similarly to
+// the server side terminal.
+const createChunk = (records: Array<Record>) => {
+	let content = '';
+	records.forEach((record) => {
+		const {t, ctx} = record;
+		let {lvl, msg} = record;
+		let color = '#ce3c23';
+		switch (lvl) {
+		case 'trace':
+		case 'trce':
+			lvl = 'TRACE';
+			color = '#3465a4';
+			break;
+		case 'debug':
+		case 'dbug':
+			lvl = 'DEBUG';
+			color = '#3d989b';
+			break;
+		case 'info':
+			lvl = 'INFO&nbsp;';
+			color = '#4c8f0f';
+			break;
+		case 'warn':
+			lvl = 'WARN&nbsp;';
+			color = '#b79a22';
+			break;
+		case 'error':
+		case 'eror':
+			lvl = 'ERROR';
+			color = '#754b70';
+			break;
+		case 'crit':
+			lvl = 'CRIT&nbsp;';
+			color = '#ce3c23';
+			break;
+		default:
+			lvl = '';
+		}
+		const time = new Date(t);
+		if (lvl === '' || !(time instanceof Date) || isNaN(time) || typeof msg !== 'string' || !Array.isArray(ctx)) {
+			content += '<span style="color:#ce3c23">Invalid log record</span><br />';
+			return;
+		}
+		if (ctx.length > 0) {
+			msg += '&nbsp;'.repeat(Math.max(40 - msg.length, 0));
+		}
+		const month = `0${time.getMonth() + 1}`.slice(-2);
+		const date = `0${time.getDate()}`.slice(-2);
+		const hours = `0${time.getHours()}`.slice(-2);
+		const minutes = `0${time.getMinutes()}`.slice(-2);
+		const seconds = `0${time.getSeconds()}`.slice(-2);
+		content += `<span style="color:${color}">${lvl}</span>[${month}-${date}|${hours}:${minutes}:${seconds}] ${msg}`;
+
+		for (let i = 0; i < ctx.length; i += 2) {
+			const key = ctx[i];
+			const val = ctx[i + 1];
+			let padding = fieldPadding.get(key);
+			if (typeof padding !== 'number' || padding < val.length) {
+				padding = val.length;
+				fieldPadding.set(key, padding);
+			}
+			let p = '';
+			if (i < ctx.length - 2) {
+				p = '&nbsp;'.repeat(padding - val.length);
+			}
+			content += ` <span style="color:${color}">${key}</span>=${val}${p}`;
+		}
+		content += '<br />';
+	});
+	return content;
+};
+
+// inserter is a state updater function for the main component, which inserts the new log chunk into the chunk array.
+// limit is the maximum length of the chunk array, used in order to prevent the browser from OOM.
+export const inserter = (limit: number) => (update: LogsMessage, prev: LogsType) => {
+	prev.topChanged = 0;
+	prev.bottomChanged = 0;
+	if (!Array.isArray(update.chunk) || update.chunk.length < 1) {
+		return prev;
+	}
+	if (!Array.isArray(prev.chunks)) {
+		prev.chunks = [];
+	}
+	const content = createChunk(update.chunk);
+	if (!update.source) {
+		// In case of stream chunk.
+		if (!prev.endBottom) {
+			return prev;
+		}
+		if (prev.chunks.length < 1) {
+			// This should never happen, because the first chunk is always a non-stream chunk.
+			return [{content, name: '00000000000000.log'}];
+		}
+		prev.chunks[prev.chunks.length - 1].content += content;
+		prev.bottomChanged = 1;
+		return prev;
+	}
+	const chunk = {
+		content,
+		name: update.source.name,
+	};
+	if (prev.chunks.length > 0 && update.source.name < prev.chunks[0].name) {
+		if (update.source.last) {
+			prev.endTop = true;
+		}
+		if (prev.chunks.length >= limit) {
+			prev.endBottom = false;
+			prev.chunks.splice(limit - 1, prev.chunks.length - limit + 1);
+			prev.bottomChanged = -1;
+		}
+		prev.chunks = [chunk, ...prev.chunks];
+		prev.topChanged = 1;
+		return prev;
+	}
+	if (update.source.last) {
+		prev.endBottom = true;
+	}
+	if (prev.chunks.length >= limit) {
+		prev.endTop = false;
+		prev.chunks.splice(0, prev.chunks.length - limit + 1);
+		prev.topChanged = -1;
+	}
+	prev.chunks = [...prev.chunks, chunk];
+	prev.bottomChanged = 1;
+	return prev;
+};
+
+// styles contains the constant styles of the component.
+const styles = {
+	logListItem: {
+		padding: 0,
+	},
+	logChunk: {
+		color:      'white',
+		fontFamily: 'monospace',
+		whiteSpace: 'nowrap',
+		width:      0,
+	},
+};
+
+export type Props = {
+	container:    Object,
+	content:      Content,
+	shouldUpdate: Object,
+	send:         string => void,
+};
+
+type State = {
+	requestAllowed: boolean,
+};
+
+// Logs renders the log page.
+class Logs extends Component<Props, State> {
+	constructor(props: Props) {
+		super(props);
+		this.content = React.createRef();
+		this.state = {
+			requestAllowed: true,
+		};
+	}
+
+	componentDidMount() {
+		const {container} = this.props;
+		container.scrollTop = container.scrollHeight - container.clientHeight;
+	}
+
+	// onScroll is triggered by the parent component's scroll event, and sends requests if the scroll position is
+	// at the top or at the bottom.
+	onScroll = () => {
+		if (!this.state.requestAllowed || typeof this.content === 'undefined') {
+			return;
+		}
+		const {logs} = this.props.content;
+		if (logs.chunks.length < 1) {
+			return;
+		}
+		if (this.atTop()) {
+			if (!logs.endTop) {
+				this.setState({requestAllowed: false});
+				this.props.send(JSON.stringify({
+					Logs: {
+						Name: logs.chunks[0].name,
+						Past: true,
+					},
+				}));
+			}
+		} else if (this.atBottom()) {
+			if (!logs.endBottom) {
+				this.setState({requestAllowed: false});
+				this.props.send(JSON.stringify({
+					Logs: {
+						Name: logs.chunks[logs.chunks.length - 1].name,
+						Past: false,
+					},
+				}));
+			}
+		}
+	};
+
+	// atTop checks if the scroll position it at the top of the container.
+	atTop = () => this.props.container.scrollTop <= this.props.container.scrollHeight * requestBand;
+
+	// atBottom checks if the scroll position it at the bottom of the container.
+	atBottom = () => {
+		const {container} = this.props;
+		return container.scrollHeight - container.scrollTop <=
+			container.clientHeight + container.scrollHeight * requestBand;
+	};
+
+	// beforeUpdate is called by the parent component, saves the previous scroll position
+	// and the height of the first log chunk, which can be deleted during the insertion.
+	beforeUpdate = () => {
+		let firstHeight = 0;
+		if (this.content && this.content.children[0] && this.content.children[0].children[0]) {
+			firstHeight = this.content.children[0].children[0].clientHeight;
+		}
+		return {
+			scrollTop: this.props.container.scrollTop,
+			firstHeight,
+		};
+	};
+
+	// didUpdate is called by the parent component, which provides the container. Sends the first request if the
+	// visible part of the container isn't full, and resets the scroll position in order to avoid jumping when new
+	// chunk is inserted.
+	didUpdate = (prevProps, prevState, snapshot) => {
+		if (typeof this.props.shouldUpdate.logs === 'undefined' || typeof this.content === 'undefined' || snapshot === null) {
+			return;
+		}
+		const {logs} = this.props.content;
+		const {container} = this.props;
+		if (typeof container === 'undefined' || logs.chunks.length < 1) {
+			return;
+		}
+		if (this.content.clientHeight < container.clientHeight) {
+			// Only enters here at the beginning, when there isn't enough log to fill the container
+			// and the scroll bar doesn't appear.
+			if (!logs.endTop) {
+				this.setState({requestAllowed: false});
+				this.props.send(JSON.stringify({
+					Logs: {
+						Name: logs.chunks[0].name,
+						Past: true,
+					},
+				}));
+			}
+			return;
+		}
+		const chunks = this.content.children[0].children;
+		let {scrollTop} = snapshot;
+		if (logs.topChanged > 0) {
+			scrollTop += chunks[0].clientHeight;
+		} else if (logs.bottomChanged > 0) {
+			if (logs.topChanged < 0) {
+				scrollTop -= snapshot.firstHeight;
+			} else if (logs.endBottom && this.atBottom()) {
+				scrollTop = container.scrollHeight - container.clientHeight;
+			}
+		}
+		container.scrollTop = scrollTop;
+		this.setState({requestAllowed: true});
+	};
+
+	render() {
+		return (
+			<div ref={(ref) => { this.content = ref; }}>
+				<List>
+					{this.props.content.logs.chunks.map((c, index) => (
+						<ListItem style={styles.logListItem} key={index}>
+							<div style={styles.logChunk} dangerouslySetInnerHTML={{__html: c.content}} />
+						</ListItem>
+					))}
+				</List>
+			</div>
+		);
+	}
+}
+
+export default Logs;

dashboard/assets/components/Main.jsx

@@ -21,6 +21,7 @@ import React, {Component} from 'react';
 import withStyles from 'material-ui/styles/withStyles';
 
 import {MENU} from '../common';
+import Logs from './Logs';
 import Footer from './Footer';
 import type {Content} from '../types/content';
 
@@ -50,10 +51,36 @@ export type Props = {
 	active:       string,
 	content:      Content,
 	shouldUpdate: Object,
+	send:         string => void,
 };
 
 // Main renders the chosen content.
 class Main extends Component<Props> {
+	constructor(props) {
+		super(props);
+		this.container = React.createRef();
+		this.content = React.createRef();
+	}
+
+	getSnapshotBeforeUpdate() {
+		if (this.content && typeof this.content.beforeUpdate === 'function') {
+			return this.content.beforeUpdate();
+		}
+		return null;
+	}
+
+	componentDidUpdate(prevProps, prevState, snapshot) {
+		if (this.content && typeof this.content.didUpdate === 'function') {
+			this.content.didUpdate(prevProps, prevState, snapshot);
+		}
+	}
+
+	onScroll = () => {
+		if (this.content && typeof this.content.onScroll === 'function') {
+			this.content.onScroll();
+		}
+	};
+
 	render() {
 		const {
 			classes, active, content, shouldUpdate,
@@ -69,12 +96,27 @@ class Main extends Component<Props> {
 			children = <div>Work in progress.</div>;
 			break;
 		case MENU.get('logs').id:
-			children = <div>{content.logs.log.map((log, index) => <div key={index}>{log}</div>)}</div>;
+			children = (
+				<Logs
+					ref={(ref) => { this.content = ref; }}
+					container={this.container}
+					send={this.props.send}
+					content={this.props.content}
+					shouldUpdate={shouldUpdate}
+				/>
+			);
 		}
 
 		return (
 			<div style={styles.wrapper}>
-				<div className={classes.content} style={styles.content}>{children}</div>
+				<div
+					className={classes.content}
+					style={styles.content}
+					ref={(ref) => { this.container = ref; }}
+					onScroll={this.onScroll}
+				>
+					{children}
+				</div>
 				<Footer
 					general={content.general}
 					system={content.system}

dashboard/assets/index.html

@@ -14,6 +14,9 @@
             ::-webkit-scrollbar-thumb {
                 background: #212121;
             }
+            ::-webkit-scrollbar-corner {
+                background: transparent;
+            }
         </style>
     </head>
     <body style="height: 100%; margin: 0">

dashboard/assets/types/content.jsx

@@ -65,6 +65,32 @@ export type System = {
 	diskWrite:      ChartEntries,
 };
 
-export type Logs = {
+export type Record = {
-	log: Array<string>,
+	t:   string,
+	lvl: Object,
+	msg: string,
+	ctx: Array<string>
+};
+
+export type Chunk = {
+	content: string,
+	name:    string,
+};
+
+export type Logs = {
+	chunks:        Array<Chunk>,
+	endTop:        boolean,
+	endBottom:     boolean,
+	topChanged:    number,
+	bottomChanged: number,
+};
+
+export type LogsMessage = {
+	source: ?LogFile,
+	chunk:  Array<Record>,
+};
+
+export type LogFile = {
+	name: string,
+	last: string,
 };

dashboard/assets/yarn.lock

@@ -2,70 +2,77 @@
 # yarn lockfile v1
 
 
-"@babel/code-frame@7.0.0-beta.40", "@babel/code-frame@^7.0.0-beta.40":
+"@babel/code-frame@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.0.0-beta.40.tgz#37e2b0cf7c56026b4b21d3927cadf81adec32ac6"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.0.0-beta.44.tgz#2a02643368de80916162be70865c97774f3adbd9"
   dependencies:
-    "@babel/highlight" "7.0.0-beta.40"
+    "@babel/highlight" "7.0.0-beta.44"
 
-"@babel/generator@7.0.0-beta.40":
+"@babel/generator@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.0.0-beta.40.tgz#ab61f9556f4f71dbd1138949c795bb9a21e302ea"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.0.0-beta.44.tgz#c7e67b9b5284afcf69b309b50d7d37f3e5033d42"
   dependencies:
-    "@babel/types" "7.0.0-beta.40"
+    "@babel/types" "7.0.0-beta.44"
     jsesc "^2.5.1"
     lodash "^4.2.0"
     source-map "^0.5.0"
     trim-right "^1.0.1"
 
-"@babel/helper-function-name@7.0.0-beta.40":
+"@babel/helper-function-name@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.0.0-beta.40.tgz#9d033341ab16517f40d43a73f2d81fc431ccd7b6"
+  resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.0.0-beta.44.tgz#e18552aaae2231100a6e485e03854bc3532d44dd"
   dependencies:
-    "@babel/helper-get-function-arity" "7.0.0-beta.40"
+    "@babel/helper-get-function-arity" "7.0.0-beta.44"
-    "@babel/template" "7.0.0-beta.40"
+    "@babel/template" "7.0.0-beta.44"
-    "@babel/types" "7.0.0-beta.40"
+    "@babel/types" "7.0.0-beta.44"
 
-"@babel/helper-get-function-arity@7.0.0-beta.40":
+"@babel/helper-get-function-arity@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.0.0-beta.40.tgz#ac0419cf067b0ec16453e1274f03878195791c6e"
+  resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.0.0-beta.44.tgz#d03ca6dd2b9f7b0b1e6b32c56c72836140db3a15"
   dependencies:
-    "@babel/types" "7.0.0-beta.40"
+    "@babel/types" "7.0.0-beta.44"
 
-"@babel/highlight@7.0.0-beta.40":
+"@babel/helper-split-export-declaration@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.0.0-beta.40.tgz#b43d67d76bf46e1d10d227f68cddcd263786b255"
+  resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.0.0-beta.44.tgz#c0b351735e0fbcb3822c8ad8db4e583b05ebd9dc"
+  dependencies:
+    "@babel/types" "7.0.0-beta.44"
+
+"@babel/highlight@7.0.0-beta.44":
+  version "7.0.0-beta.44"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.0.0-beta.44.tgz#18c94ce543916a80553edcdcf681890b200747d5"
   dependencies:
     chalk "^2.0.0"
     esutils "^2.0.2"
     js-tokens "^3.0.0"
 
-"@babel/template@7.0.0-beta.40":
+"@babel/template@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.0.0-beta.40.tgz#034988c6424eb5c3268fe6a608626de1f4410fc8"
+  resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.0.0-beta.44.tgz#f8832f4fdcee5d59bf515e595fc5106c529b394f"
   dependencies:
-    "@babel/code-frame" "7.0.0-beta.40"
+    "@babel/code-frame" "7.0.0-beta.44"
-    "@babel/types" "7.0.0-beta.40"
+    "@babel/types" "7.0.0-beta.44"
-    babylon "7.0.0-beta.40"
+    babylon "7.0.0-beta.44"
     lodash "^4.2.0"
 
-"@babel/traverse@^7.0.0-beta.40":
+"@babel/traverse@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.0.0-beta.40.tgz#d140e449b2e093ef9fe1a2eecc28421ffb4e521e"
+  resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.0.0-beta.44.tgz#a970a2c45477ad18017e2e465a0606feee0d2966"
   dependencies:
-    "@babel/code-frame" "7.0.0-beta.40"
+    "@babel/code-frame" "7.0.0-beta.44"
-    "@babel/generator" "7.0.0-beta.40"
+    "@babel/generator" "7.0.0-beta.44"
-    "@babel/helper-function-name" "7.0.0-beta.40"
+    "@babel/helper-function-name" "7.0.0-beta.44"
-    "@babel/types" "7.0.0-beta.40"
+    "@babel/helper-split-export-declaration" "7.0.0-beta.44"
-    babylon "7.0.0-beta.40"
+    "@babel/types" "7.0.0-beta.44"
-    debug "^3.0.1"
+    babylon "7.0.0-beta.44"
+    debug "^3.1.0"
     globals "^11.1.0"
     invariant "^2.2.0"
     lodash "^4.2.0"
 
-"@babel/types@7.0.0-beta.40", "@babel/types@^7.0.0-beta.40":
+"@babel/types@7.0.0-beta.44":
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.0.0-beta.40.tgz#25c3d7aae14126abe05fcb098c65a66b6d6b8c14"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.0.0-beta.44.tgz#6b1b164591f77dec0a0342aca995f2d046b3a757"
   dependencies:
     esutils "^2.0.2"
     lodash "^4.2.0"
@@ -376,8 +383,8 @@ babel-code-frame@^6.22.0, babel-code-frame@^6.26.0:
     js-tokens "^3.0.2"
 
 babel-core@^6.26.0:
-  version "6.26.0"
+  version "6.26.3"
-  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.26.0.tgz#af32f78b31a6fcef119c87b0fd8d9753f03a0bb8"
+  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.26.3.tgz#b2e2f09e342d0f0c88e2f02e067794125e75c207"
   dependencies:
     babel-code-frame "^6.26.0"
     babel-generator "^6.26.0"
@@ -389,24 +396,24 @@ babel-core@^6.26.0:
     babel-traverse "^6.26.0"
     babel-types "^6.26.0"
     babylon "^6.18.0"
-    convert-source-map "^1.5.0"
+    convert-source-map "^1.5.1"
-    debug "^2.6.8"
+    debug "^2.6.9"
     json5 "^0.5.1"
     lodash "^4.17.4"
     minimatch "^3.0.4"
     path-is-absolute "^1.0.1"
-    private "^0.1.7"
+    private "^0.1.8"
     slash "^1.0.0"
-    source-map "^0.5.6"
+    source-map "^0.5.7"
 
 babel-eslint@^8.2.1:
-  version "8.2.2"
+  version "8.2.3"
-  resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-8.2.2.tgz#1102273354c6f0b29b4ea28a65f97d122296b68b"
+  resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-8.2.3.tgz#1a2e6681cc9bc4473c32899e59915e19cd6733cf"
   dependencies:
-    "@babel/code-frame" "^7.0.0-beta.40"
+    "@babel/code-frame" "7.0.0-beta.44"
-    "@babel/traverse" "^7.0.0-beta.40"
+    "@babel/traverse" "7.0.0-beta.44"
-    "@babel/types" "^7.0.0-beta.40"
+    "@babel/types" "7.0.0-beta.44"
-    babylon "^7.0.0-beta.40"
+    babylon "7.0.0-beta.44"
     eslint-scope "~3.7.1"
     eslint-visitor-keys "^1.0.0"
 
@@ -550,8 +557,8 @@ babel-helpers@^6.24.1:
     babel-template "^6.24.1"
 
 babel-loader@^7.1.2:
-  version "7.1.3"
+  version "7.1.4"
-  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-7.1.3.tgz#ff5b440da716e9153abb946251a9ab7670037b16"
+  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-7.1.4.tgz#e3463938bd4e6d55d1c174c5485d406a188ed015"
   dependencies:
     find-cache-dir "^1.0.0"
     loader-utils "^1.0.2"
@@ -1081,9 +1088,9 @@ babel-types@^6.19.0, babel-types@^6.24.1, babel-types@^6.26.0:
     lodash "^4.17.4"
     to-fast-properties "^1.0.3"
 
-babylon@7.0.0-beta.40, babylon@^7.0.0-beta.40:
+babylon@7.0.0-beta.44:
-  version "7.0.0-beta.40"
+  version "7.0.0-beta.44"
-  resolved "https://registry.yarnpkg.com/babylon/-/babylon-7.0.0-beta.40.tgz#91fc8cd56d5eb98b28e6fde41045f2957779940a"
+  resolved "https://registry.yarnpkg.com/babylon/-/babylon-7.0.0-beta.44.tgz#89159e15e6e30c5096e22d738d8c0af8a0e8ca1d"
 
 babylon@^6.18.0:
   version "6.18.0"
@@ -1413,7 +1420,15 @@ chalk@^1.1.3:
     strip-ansi "^3.0.0"
     supports-color "^2.0.0"
 
-chalk@^2.0.0, chalk@^2.1.0, chalk@^2.3.1:
+chalk@^2.0.0:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.1.tgz#18c49ab16a037b6eb0152cc83e3471338215b66e"
+  dependencies:
+    ansi-styles "^3.2.1"
+    escape-string-regexp "^1.0.5"
+    supports-color "^5.3.0"
+
+chalk@^2.1.0, chalk@^2.3.1:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.3.2.tgz#250dc96b07491bfd601e648d66ddf5f60c7a5c65"
   dependencies:
@@ -1646,7 +1661,7 @@ content-type@~1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
 
-convert-source-map@^1.5.0:
+convert-source-map@^1.5.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.1.tgz#b8278097b9bc229365de5c62cf5fcaed8b5599e5"
 
@@ -1671,8 +1686,8 @@ core-js@^1.0.0:
   resolved "https://registry.yarnpkg.com/core-js/-/core-js-1.2.7.tgz#652294c14651db28fa93bd2d5ff2983a4f08c636"
 
 core-js@^2.4.0, core-js@^2.5.0:
-  version "2.5.3"
+  version "2.5.7"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.3.tgz#8acc38345824f16d8365b7c9b4259168e8ed603e"
+  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.7.tgz#f972608ff0cead68b841a16a932d0b183791814e"
 
 core-util-is@1.0.2, core-util-is@~1.0.0:
   version "1.0.2"
@@ -1914,7 +1929,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.6, debug@^2.6.8, debug@^2.6.
   dependencies:
     ms "2.0.0"
 
-debug@^3.0.1, debug@^3.1.0:
+debug@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
   dependencies:
@@ -2916,10 +2931,14 @@ global@~4.3.0:
     min-document "^2.19.0"
     process "~0.5.1"
 
-globals@^11.0.1, globals@^11.1.0:
+globals@^11.0.1:
   version "11.3.0"
   resolved "https://registry.yarnpkg.com/globals/-/globals-11.3.0.tgz#e04fdb7b9796d8adac9c8f64c14837b2313378b0"
 
+globals@^11.1.0:
+  version "11.5.0"
+  resolved "https://registry.yarnpkg.com/globals/-/globals-11.5.0.tgz#6bc840de6771173b191f13d3a9c94d441ee92642"
+
 globals@^9.18.0:
   version "9.18.0"
   resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a"
@@ -3176,10 +3195,16 @@ hyphenate-style-name@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/hyphenate-style-name/-/hyphenate-style-name-1.0.2.tgz#31160a36930adaf1fc04c6074f7eb41465d4ec4b"
 
-iconv-lite@0.4.19, iconv-lite@^0.4.17, iconv-lite@~0.4.13:
+iconv-lite@0.4.19, iconv-lite@^0.4.17:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b"
 
+iconv-lite@~0.4.13:
+  version "0.4.23"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
+  dependencies:
+    safer-buffer ">= 2.1.2 < 3"
+
 icss-replace-symbols@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz#06ea6f83679a7749e386cfe1fe812ae5db223ded"
@@ -3272,8 +3297,8 @@ interpret@^1.0.0:
   resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.1.0.tgz#7ed1b1410c6a0e0f78cf95d3b8440c63f78b8614"
 
 invariant@^2.2.0, invariant@^2.2.2:
-  version "2.2.3"
+  version "2.2.4"
-  resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.3.tgz#1a827dfde7dcbd7c323f0ca826be8fa7c5e9d688"
+  resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6"
   dependencies:
     loose-envify "^1.0.0"
 
@@ -3863,10 +3888,14 @@ lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
 
-lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.2, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0, lodash@~4.17.4:
+lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.2, lodash@^4.3.0, lodash@~4.17.4:
   version "4.17.5"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511"
 
+lodash@^4.17.4, lodash@^4.2.0:
+  version "4.17.10"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7"
+
 loglevel@^1.4.1:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.6.1.tgz#e0fc95133b6ef276cdc8887cdaf24aa6f156f8fa"
@@ -3904,8 +3933,8 @@ macaddress@^0.2.8:
   resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.8.tgz#5904dc537c39ec6dbefeae902327135fa8511f12"
 
 make-dir@^1.0.0:
-  version "1.2.0"
+  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.2.0.tgz#6d6a49eead4aae296c53bbf3a1a008bd6c89469b"
+  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
   dependencies:
     pify "^3.0.0"
 
@@ -4895,7 +4924,7 @@ preserve@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b"
 
-private@^0.1.6, private@^0.1.7:
+private@^0.1.6, private@^0.1.8:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
 
@@ -5061,8 +5090,8 @@ rc@^1.1.7:
     strip-json-comments "~2.0.1"
 
 react-dom@^16.2.0:
-  version "16.2.0"
+  version "16.4.0"
-  resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.2.0.tgz#69003178601c0ca19b709b33a83369fe6124c044"
+  resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.4.0.tgz#099f067dd5827ce36a29eaf9a6cdc7cbf6216b1e"
   dependencies:
     fbjs "^0.8.16"
     loose-envify "^1.1.0"
@@ -5138,8 +5167,8 @@ react-transition-group@^2.2.1:
     warning "^3.0.0"
 
 react@^16.2.0:
-  version "16.2.0"
+  version "16.4.0"
-  resolved "https://registry.yarnpkg.com/react/-/react-16.2.0.tgz#a31bd2dab89bff65d42134fa187f24d054c273ba"
+  resolved "https://registry.yarnpkg.com/react/-/react-16.4.0.tgz#402c2db83335336fba1962c08b98c6272617d585"
   dependencies:
     fbjs "^0.8.16"
     loose-envify "^1.1.0"
@@ -5471,6 +5500,10 @@ safe-regex@^1.1.0:
   dependencies:
     ret "~0.1.10"
 
+"safer-buffer@>= 2.1.2 < 3":
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
+
 sax@~1.2.1:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -5914,12 +5947,18 @@ supports-color@^4.2.1:
   dependencies:
     has-flag "^2.0.0"
 
-supports-color@^5.1.0, supports-color@^5.2.0, supports-color@^5.3.0:
+supports-color@^5.1.0, supports-color@^5.2.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.3.0.tgz#5b24ac15db80fa927cf5227a4a33fd3c4c7676c0"
   dependencies:
     has-flag "^3.0.0"
 
+supports-color@^5.3.0:
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54"
+  dependencies:
+    has-flag "^3.0.0"
+
 svgo@^0.7.0:
   version "0.7.2"
   resolved "https://registry.yarnpkg.com/svgo/-/svgo-0.7.2.tgz#9f5772413952135c6fefbf40afe6a4faa88b4bb5"
@@ -6108,8 +6147,8 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
 
 ua-parser-js@^0.7.9:
-  version "0.7.17"
+  version "0.7.18"
-  resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.17.tgz#e9ec5f9498b9ec910e7ae3ac626a805c4d09ecac"
+  resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.18.tgz#a7bfd92f56edfb117083b69e31d2aa8882d4b1ed"
 
 uglify-js@^2.8.29:
   version "2.8.29"
@@ -6395,8 +6434,8 @@ websocket-extensions@>=0.1.1:
   resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29"
 
 whatwg-fetch@>=0.10.0:
-  version "2.0.3"
+  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz#9c84ec2dcf68187ff00bc64e1274b442176e1c84"
+  resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz#dde6a5df315f9d39991aa17621853d720b85566f"
 
 whet.extend@~0.9.9:
   version "0.9.9"

dashboard/dashboard.go

@@ -32,12 +32,15 @@ import (
 	"sync/atomic"
 	"time"
 
+	"io"
+
 	"github.com/elastic/gosigar"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/mohae/deepcopy"
 	"golang.org/x/net/websocket"
 )
 
@@ -60,10 +63,11 @@ type Dashboard struct {
 
 	listener net.Listener
 	conns    map[uint32]*client // Currently live websocket connections
-	charts   *SystemMessage
+	history  *Message
-	commit   string
 	lock     sync.RWMutex // Lock protecting the dashboard's internals
 
+	logdir string
+
 	quit chan chan error // Channel used for graceful exit
 	wg   sync.WaitGroup
 }
@@ -71,18 +75,27 @@ type Dashboard struct {
 // client represents active websocket connection with a remote browser.
 type client struct {
 	conn   *websocket.Conn // Particular live websocket connection
-	msg    chan Message    // Message queue for the update messages
+	msg    chan *Message   // Message queue for the update messages
 	logger log.Logger      // Logger for the particular live websocket connection
 }
 
 // New creates a new dashboard instance with the given configuration.
-func New(config *Config, commit string) (*Dashboard, error) {
+func New(config *Config, commit string, logdir string) *Dashboard {
 	now := time.Now()
-	db := &Dashboard{
+	versionMeta := ""
+	if len(params.VersionMeta) > 0 {
+		versionMeta = fmt.Sprintf(" (%s)", params.VersionMeta)
+	}
+	return &Dashboard{
 		conns:  make(map[uint32]*client),
 		config: config,
 		quit:   make(chan chan error),
-		charts: &SystemMessage{
+		history: &Message{
+			General: &GeneralMessage{
+				Commit:  commit,
+				Version: fmt.Sprintf("v%d.%d.%d%s", params.VersionMajor, params.VersionMinor, params.VersionPatch, versionMeta),
+			},
+			System: &SystemMessage{
 				ActiveMemory:   emptyChartEntries(now, activeMemorySampleLimit, config.Refresh),
 				VirtualMemory:  emptyChartEntries(now, virtualMemorySampleLimit, config.Refresh),
 				NetworkIngress: emptyChartEntries(now, networkIngressSampleLimit, config.Refresh),
@@ -92,9 +105,9 @@ func New(config *Config, commit string) (*Dashboard, error) {
 				DiskRead:       emptyChartEntries(now, diskReadSampleLimit, config.Refresh),
 				DiskWrite:      emptyChartEntries(now, diskWriteSampleLimit, config.Refresh),
 			},
-		commit: commit,
+		},
+		logdir: logdir,
 	}
-	return db, nil
 }
 
 // emptyChartEntries returns a ChartEntry array containing limit number of empty samples.
@@ -108,19 +121,20 @@ func emptyChartEntries(t time.Time, limit int, refresh time.Duration) ChartEntri
 	return ce
 }
 
-// Protocols is a meaningless implementation of node.Service.
+// Protocols implements the node.Service interface.
 func (db *Dashboard) Protocols() []p2p.Protocol { return nil }
 
-// APIs is a meaningless implementation of node.Service.
+// APIs implements the node.Service interface.
 func (db *Dashboard) APIs() []rpc.API { return nil }
 
-// Start implements node.Service, starting the data collection thread and the listening server of the dashboard.
+// Start starts the data collection thread and the listening server of the dashboard.
+// Implements the node.Service interface.
 func (db *Dashboard) Start(server *p2p.Server) error {
 	log.Info("Starting dashboard")
 
 	db.wg.Add(2)
 	go db.collectData()
-	go db.collectLogs() // In case of removing this line change 2 back to 1 in wg.Add.
+	go db.streamLogs()
 
 	http.HandleFunc("/", db.webHandler)
 	http.Handle("/api", websocket.Handler(db.apiHandler))
@@ -136,7 +150,8 @@ func (db *Dashboard) Start(server *p2p.Server) error {
 	return nil
 }
 
-// Stop implements node.Service, stopping the data collection thread and the connection listener of the dashboard.
+// Stop stops the data collection thread and the connection listener of the dashboard.
+// Implements the node.Service interface.
 func (db *Dashboard) Stop() error {
 	// Close the connection listener.
 	var errs []error
@@ -194,7 +209,7 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) {
 	id := atomic.AddUint32(&nextID, 1)
 	client := &client{
 		conn:   conn,
-		msg:    make(chan Message, 128),
+		msg:    make(chan *Message, 128),
 		logger: log.New("id", id),
 	}
 	done := make(chan struct{})
@@ -218,29 +233,10 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) {
 		}
 	}()
 
-	versionMeta := ""
-	if len(params.VersionMeta) > 0 {
-		versionMeta = fmt.Sprintf(" (%s)", params.VersionMeta)
-	}
-	// Send the past data.
-	client.msg <- Message{
-		General: &GeneralMessage{
-			Version: fmt.Sprintf("v%d.%d.%d%s", params.VersionMajor, params.VersionMinor, params.VersionPatch, versionMeta),
-			Commit:  db.commit,
-		},
-		System: &SystemMessage{
-			ActiveMemory:   db.charts.ActiveMemory,
-			VirtualMemory:  db.charts.VirtualMemory,
-			NetworkIngress: db.charts.NetworkIngress,
-			NetworkEgress:  db.charts.NetworkEgress,
-			ProcessCPU:     db.charts.ProcessCPU,
-			SystemCPU:      db.charts.SystemCPU,
-			DiskRead:       db.charts.DiskRead,
-			DiskWrite:      db.charts.DiskWrite,
-		},
-	}
-	// Start tracking the connection and drop at connection loss.
 	db.lock.Lock()
+	// Send the past data.
+	client.msg <- deepcopy.Copy(db.history).(*Message)
+	// Start tracking the connection and drop at connection loss.
 	db.conns[id] = client
 	db.lock.Unlock()
 	defer func() {
@@ -249,29 +245,53 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) {
 		db.lock.Unlock()
 	}()
 	for {
-		fail := []byte{}
+		r := new(Request)
-		if _, err := conn.Read(fail); err != nil {
+		if err := websocket.JSON.Receive(conn, r); err != nil {
+			if err != io.EOF {
+				client.logger.Warn("Failed to receive request", "err", err)
+			}
 			close(done)
 			return
 		}
-		// Ignore all messages
+		if r.Logs != nil {
+			db.handleLogRequest(r.Logs, client)
+		}
+	}
+}
+
+// meterCollector returns a function, which retrieves a specific meter.
+func meterCollector(name string) func() int64 {
+	if metric := metrics.DefaultRegistry.Get(name); metric != nil {
+		m := metric.(metrics.Meter)
+		return func() int64 {
+			return m.Count()
+		}
+	}
+	return func() int64 {
+		return 0
 	}
 }
 
 // collectData collects the required data to plot on the dashboard.
 func (db *Dashboard) collectData() {
 	defer db.wg.Done()
+
 	systemCPUUsage := gosigar.Cpu{}
 	systemCPUUsage.Get()
 	var (
 		mem runtime.MemStats
 
-		prevNetworkIngress = metrics.DefaultRegistry.Get("p2p/InboundTraffic").(metrics.Meter).Count()
+		collectNetworkIngress = meterCollector("p2p/InboundTraffic")
-		prevNetworkEgress  = metrics.DefaultRegistry.Get("p2p/OutboundTraffic").(metrics.Meter).Count()
+		collectNetworkEgress  = meterCollector("p2p/OutboundTraffic")
+		collectDiskRead       = meterCollector("eth/db/chaindata/disk/read")
+		collectDiskWrite      = meterCollector("eth/db/chaindata/disk/write")
+
+		prevNetworkIngress = collectNetworkIngress()
+		prevNetworkEgress  = collectNetworkEgress()
 		prevProcessCPUTime = getProcessCPUTime()
 		prevSystemCPUUsage = systemCPUUsage
-		prevDiskRead       = metrics.DefaultRegistry.Get("eth/db/chaindata/disk/read").(metrics.Meter).Count()
+		prevDiskRead       = collectDiskRead()
-		prevDiskWrite      = metrics.DefaultRegistry.Get("eth/db/chaindata/disk/write").(metrics.Meter).Count()
+		prevDiskWrite      = collectDiskWrite()
 
 		frequency = float64(db.config.Refresh / time.Second)
 		numCPU    = float64(runtime.NumCPU())
@@ -285,12 +305,12 @@ func (db *Dashboard) collectData() {
 		case <-time.After(db.config.Refresh):
 			systemCPUUsage.Get()
 			var (
-				curNetworkIngress = metrics.DefaultRegistry.Get("p2p/InboundTraffic").(metrics.Meter).Count()
+				curNetworkIngress = collectNetworkIngress()
-				curNetworkEgress  = metrics.DefaultRegistry.Get("p2p/OutboundTraffic").(metrics.Meter).Count()
+				curNetworkEgress  = collectNetworkEgress()
 				curProcessCPUTime = getProcessCPUTime()
 				curSystemCPUUsage = systemCPUUsage
-				curDiskRead       = metrics.DefaultRegistry.Get("eth/db/chaindata/disk/read").(metrics.Meter).Count()
+				curDiskRead       = collectDiskRead()
-				curDiskWrite      = metrics.DefaultRegistry.Get("eth/db/chaindata/disk/write").(metrics.Meter).Count()
+				curDiskWrite      = collectDiskWrite()
 
 				deltaNetworkIngress = float64(curNetworkIngress - prevNetworkIngress)
 				deltaNetworkEgress  = float64(curNetworkEgress - prevNetworkEgress)
@@ -341,14 +361,17 @@ func (db *Dashboard) collectData() {
 				Time:  now,
 				Value: float64(deltaDiskWrite) / frequency,
 			}
-			db.charts.ActiveMemory = append(db.charts.ActiveMemory[1:], activeMemory)
+			sys := db.history.System
-			db.charts.VirtualMemory = append(db.charts.VirtualMemory[1:], virtualMemory)
+			db.lock.Lock()
-			db.charts.NetworkIngress = append(db.charts.NetworkIngress[1:], networkIngress)
+			sys.ActiveMemory = append(sys.ActiveMemory[1:], activeMemory)
-			db.charts.NetworkEgress = append(db.charts.NetworkEgress[1:], networkEgress)
+			sys.VirtualMemory = append(sys.VirtualMemory[1:], virtualMemory)
-			db.charts.ProcessCPU = append(db.charts.ProcessCPU[1:], processCPU)
+			sys.NetworkIngress = append(sys.NetworkIngress[1:], networkIngress)
-			db.charts.SystemCPU = append(db.charts.SystemCPU[1:], systemCPU)
+			sys.NetworkEgress = append(sys.NetworkEgress[1:], networkEgress)
-			db.charts.DiskRead = append(db.charts.DiskRead[1:], diskRead)
+			sys.ProcessCPU = append(sys.ProcessCPU[1:], processCPU)
-			db.charts.DiskWrite = append(db.charts.DiskRead[1:], diskWrite)
+			sys.SystemCPU = append(sys.SystemCPU[1:], systemCPU)
+			sys.DiskRead = append(sys.DiskRead[1:], diskRead)
+			sys.DiskWrite = append(sys.DiskRead[1:], diskWrite)
+			db.lock.Unlock()
 
 			db.sendToAll(&Message{
 				System: &SystemMessage{
@@ -366,34 +389,12 @@ func (db *Dashboard) collectData() {
 	}
 }
 
-// collectLogs collects and sends the logs to the active dashboards.
-func (db *Dashboard) collectLogs() {
-	defer db.wg.Done()
-
-	id := 1
-	// TODO (kurkomisi): log collection comes here.
-	for {
-		select {
-		case errc := <-db.quit:
-			errc <- nil
-			return
-		case <-time.After(db.config.Refresh / 2):
-			db.sendToAll(&Message{
-				Logs: &LogsMessage{
-					Log: []string{fmt.Sprintf("%-4d: This is a fake log.", id)},
-				},
-			})
-			id++
-		}
-	}
-}
-
 // sendToAll sends the given message to the active dashboards.
 func (db *Dashboard) sendToAll(msg *Message) {
 	db.lock.Lock()
 	for _, c := range db.conns {
 		select {
-		case c.msg <- *msg:
+		case c.msg <- msg:
 		default:
 			c.conn.Close()
 		}

dashboard/log.go

@@ -0,0 +1,288 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package dashboard
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/mohae/deepcopy"
+	"github.com/rjeczalik/notify"
+)
+
+var emptyChunk = json.RawMessage("[]")
+
+// prepLogs creates a JSON array from the given log record buffer.
+// Returns the prepared array and the position of the last '\n'
+// character in the original buffer, or -1 if it doesn't contain any.
+func prepLogs(buf []byte) (json.RawMessage, int) {
+	b := make(json.RawMessage, 1, len(buf)+1)
+	b[0] = '['
+	b = append(b, buf...)
+	last := -1
+	for i := 1; i < len(b); i++ {
+		if b[i] == '\n' {
+			b[i] = ','
+			last = i
+		}
+	}
+	if last < 0 {
+		return emptyChunk, -1
+	}
+	b[last] = ']'
+	return b[:last+1], last - 1
+}
+
+// handleLogRequest searches for the log file specified by the timestamp of the
+// request, creates a JSON array out of it and sends it to the requesting client.
+func (db *Dashboard) handleLogRequest(r *LogsRequest, c *client) {
+	files, err := ioutil.ReadDir(db.logdir)
+	if err != nil {
+		log.Warn("Failed to open logdir", "path", db.logdir, "err", err)
+		return
+	}
+	re := regexp.MustCompile(`\.log$`)
+	fileNames := make([]string, 0, len(files))
+	for _, f := range files {
+		if f.Mode().IsRegular() && re.MatchString(f.Name()) {
+			fileNames = append(fileNames, f.Name())
+		}
+	}
+	if len(fileNames) < 1 {
+		log.Warn("No log files in logdir", "path", db.logdir)
+		return
+	}
+	idx := sort.Search(len(fileNames), func(idx int) bool {
+		// Returns the smallest index such as fileNames[idx] >= r.Name,
+		// if there is no such index, returns n.
+		return fileNames[idx] >= r.Name
+	})
+
+	switch {
+	case idx < 0:
+		return
+	case idx == 0 && r.Past:
+		return
+	case idx >= len(fileNames):
+		return
+	case r.Past:
+		idx--
+	case idx == len(fileNames)-1 && fileNames[idx] == r.Name:
+		return
+	case idx == len(fileNames)-1 || (idx == len(fileNames)-2 && fileNames[idx] == r.Name):
+		// The last file is continuously updated, and its chunks are streamed,
+		// so in order to avoid log record duplication on the client side, it is
+		// handled differently. Its actual content is always saved in the history.
+		db.lock.Lock()
+		if db.history.Logs != nil {
+			c.msg <- &Message{
+				Logs: db.history.Logs,
+			}
+		}
+		db.lock.Unlock()
+		return
+	case fileNames[idx] == r.Name:
+		idx++
+	}
+
+	path := filepath.Join(db.logdir, fileNames[idx])
+	var buf []byte
+	if buf, err = ioutil.ReadFile(path); err != nil {
+		log.Warn("Failed to read file", "path", path, "err", err)
+		return
+	}
+	chunk, end := prepLogs(buf)
+	if end < 0 {
+		log.Warn("The file doesn't contain valid logs", "path", path)
+		return
+	}
+	c.msg <- &Message{
+		Logs: &LogsMessage{
+			Source: &LogFile{
+				Name: fileNames[idx],
+				Last: r.Past && idx == 0,
+			},
+			Chunk: chunk,
+		},
+	}
+}
+
+// streamLogs watches the file system, and when the logger writes
+// the new log records into the files, picks them up, then makes
+// JSON array out of them and sends them to the clients.
+func (db *Dashboard) streamLogs() {
+	defer db.wg.Done()
+	var (
+		err  error
+		errc chan error
+	)
+	defer func() {
+		if errc == nil {
+			errc = <-db.quit
+		}
+		errc <- err
+	}()
+
+	files, err := ioutil.ReadDir(db.logdir)
+	if err != nil {
+		log.Warn("Failed to open logdir", "path", db.logdir, "err", err)
+		return
+	}
+	var (
+		opened *os.File // File descriptor for the opened active log file.
+		buf    []byte   // Contains the recently written log chunks, which are not sent to the clients yet.
+	)
+
+	// The log records are always written into the last file in alphabetical order, because of the timestamp.
+	re := regexp.MustCompile(`\.log$`)
+	i := len(files) - 1
+	for i >= 0 && (!files[i].Mode().IsRegular() || !re.MatchString(files[i].Name())) {
+		i--
+	}
+	if i < 0 {
+		log.Warn("No log files in logdir", "path", db.logdir)
+		return
+	}
+	if opened, err = os.OpenFile(filepath.Join(db.logdir, files[i].Name()), os.O_RDONLY, 0600); err != nil {
+		log.Warn("Failed to open file", "name", files[i].Name(), "err", err)
+		return
+	}
+	defer opened.Close() // Close the lastly opened file.
+	fi, err := opened.Stat()
+	if err != nil {
+		log.Warn("Problem with file", "name", opened.Name(), "err", err)
+		return
+	}
+	db.lock.Lock()
+	db.history.Logs = &LogsMessage{
+		Source: &LogFile{
+			Name: fi.Name(),
+			Last: true,
+		},
+		Chunk: emptyChunk,
+	}
+	db.lock.Unlock()
+
+	watcher := make(chan notify.EventInfo, 10)
+	if err := notify.Watch(db.logdir, watcher, notify.Create); err != nil {
+		log.Warn("Failed to create file system watcher", "err", err)
+		return
+	}
+	defer notify.Stop(watcher)
+
+	ticker := time.NewTicker(db.config.Refresh)
+	defer ticker.Stop()
+
+loop:
+	for err == nil || errc == nil {
+		select {
+		case event := <-watcher:
+			// Make sure that new log file was created.
+			if !re.Match([]byte(event.Path())) {
+				break
+			}
+			if opened == nil {
+				log.Warn("The last log file is not opened")
+				break loop
+			}
+			// The new log file's name is always greater,
+			// because it is created using the actual log record's time.
+			if opened.Name() >= event.Path() {
+				break
+			}
+			// Read the rest of the previously opened file.
+			chunk, err := ioutil.ReadAll(opened)
+			if err != nil {
+				log.Warn("Failed to read file", "name", opened.Name(), "err", err)
+				break loop
+			}
+			buf = append(buf, chunk...)
+			opened.Close()
+
+			if chunk, last := prepLogs(buf); last >= 0 {
+				// Send the rest of the previously opened file.
+				db.sendToAll(&Message{
+					Logs: &LogsMessage{
+						Chunk: chunk,
+					},
+				})
+			}
+			if opened, err = os.OpenFile(event.Path(), os.O_RDONLY, 0644); err != nil {
+				log.Warn("Failed to open file", "name", event.Path(), "err", err)
+				break loop
+			}
+			buf = buf[:0]
+
+			// Change the last file in the history.
+			fi, err := opened.Stat()
+			if err != nil {
+				log.Warn("Problem with file", "name", opened.Name(), "err", err)
+				break loop
+			}
+			db.lock.Lock()
+			db.history.Logs.Source.Name = fi.Name()
+			db.history.Logs.Chunk = emptyChunk
+			db.lock.Unlock()
+		case <-ticker.C: // Send log updates to the client.
+			if opened == nil {
+				log.Warn("The last log file is not opened")
+				break loop
+			}
+			// Read the new logs created since the last read.
+			chunk, err := ioutil.ReadAll(opened)
+			if err != nil {
+				log.Warn("Failed to read file", "name", opened.Name(), "err", err)
+				break loop
+			}
+			b := append(buf, chunk...)
+
+			chunk, last := prepLogs(b)
+			if last < 0 {
+				break
+			}
+			// Only keep the invalid part of the buffer, which can be valid after the next read.
+			buf = b[last+1:]
+
+			var l *LogsMessage
+			// Update the history.
+			db.lock.Lock()
+			if bytes.Equal(db.history.Logs.Chunk, emptyChunk) {
+				db.history.Logs.Chunk = chunk
+				l = deepcopy.Copy(db.history.Logs).(*LogsMessage)
+			} else {
+				b = make([]byte, len(db.history.Logs.Chunk)+len(chunk)-1)
+				copy(b, db.history.Logs.Chunk)
+				b[len(db.history.Logs.Chunk)-1] = ','
+				copy(b[len(db.history.Logs.Chunk):], chunk[1:])
+				db.history.Logs.Chunk = b
+				l = &LogsMessage{Chunk: chunk}
+			}
+			db.lock.Unlock()
+
+			db.sendToAll(&Message{Logs: l})
+		case errc = <-db.quit:
+			break loop
+		}
+	}
+}

dashboard/message.go

@@ -16,7 +16,10 @@
 
 package dashboard
 
-import "time"
+import (
+	"encoding/json"
+	"time"
+)
 
 type Message struct {
 	General *GeneralMessage `json:"general,omitempty"`
@@ -67,6 +70,24 @@ type SystemMessage struct {
 	DiskWrite      ChartEntries `json:"diskWrite,omitempty"`
 }
 
+// LogsMessage wraps up a log chunk. If Source isn't present, the chunk is a stream chunk.
 type LogsMessage struct {
-	Log []string `json:"log,omitempty"`
+	Source *LogFile        `json:"source,omitempty"` // Attributes of the log file.
+	Chunk  json.RawMessage `json:"chunk"`            // Contains log records.
+}
+
+// LogFile contains the attributes of a log file.
+type LogFile struct {
+	Name string `json:"name"` // The name of the file.
+	Last bool   `json:"last"` // Denotes if the actual log file is the last one in the directory.
+}
+
+// Request represents the client request.
+type Request struct {
+	Logs *LogsRequest `json:"logs,omitempty"`
+}
+
+type LogsRequest struct {
+	Name string `json:"name"` // The request handler searches for log file based on this file name.
+	Past bool   `json:"past"` // Denotes whether the client wants the previous or the next file.
 }

ethdb/database.go

@@ -34,9 +34,7 @@ import (
 )
 
 const (
-	writeDelayNThreshold       = 200
+	writePauseWarningThrottler = 1 * time.Minute
-	writeDelayThreshold        = 350 * time.Millisecond
-	writeDelayWarningThrottler = 1 * time.Minute
 )
 
 var OpenFileLimit = 64
@@ -206,8 +204,6 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 	// Create storage and warning log tracer for write delay.
 	var (
 		delaystats      [2]int64
-		lastWriteDelay  time.Time
-		lastWriteDelayN time.Time
 		lastWritePaused time.Time
 	)
 
@@ -293,36 +289,17 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 		}
 		if db.writeDelayNMeter != nil {
 			db.writeDelayNMeter.Mark(delayN - delaystats[0])
-			// If the write delay number been collected in the last minute exceeds the predefined threshold,
-			// print a warning log here.
-			// If a warning that db performance is laggy has been displayed,
-			// any subsequent warnings will be withhold for 1 minute to don't overwhelm the user.
-			if int(db.writeDelayNMeter.Rate1()) > writeDelayNThreshold &&
-				time.Now().After(lastWriteDelayN.Add(writeDelayWarningThrottler)) {
-				db.log.Warn("Write delay number exceeds the threshold (200 per second) in the last minute")
-				lastWriteDelayN = time.Now()
-			}
 		}
 		if db.writeDelayMeter != nil {
 			db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1])
-			// If the write delay duration been collected in the last minute exceeds the predefined threshold,
-			// print a warning log here.
-			// If a warning that db performance is laggy has been displayed,
-			// any subsequent warnings will be withhold for 1 minute to don't overwhelm the user.
-			if int64(db.writeDelayMeter.Rate1()) > writeDelayThreshold.Nanoseconds() &&
-				time.Now().After(lastWriteDelay.Add(writeDelayWarningThrottler)) {
-				db.log.Warn("Write delay duration exceeds the threshold (35% of the time) in the last minute")
-				lastWriteDelay = time.Now()
-			}
 		}
 		// If a warning that db is performing compaction has been displayed, any subsequent
 		// warnings will be withheld for one minute not to overwhelm the user.
 		if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 &&
-			time.Now().After(lastWritePaused.Add(writeDelayWarningThrottler)) {
+			time.Now().After(lastWritePaused.Add(writePauseWarningThrottler)) {
 			db.log.Warn("Database compacting, degraded performance")
 			lastWritePaused = time.Now()
 		}
-
 		delaystats[0], delaystats[1] = delayN, duration.Nanoseconds()
 
 		// Retrieve the database iostats.

internal/debug/flags.go

@@ -95,7 +95,10 @@ var Flags = []cli.Flag{
 	memprofilerateFlag, blockprofilerateFlag, cpuprofileFlag, traceFlag,
 }
 
-var glogger *log.GlogHandler
+var (
+	ostream log.Handler
+	glogger *log.GlogHandler
+)
 
 func init() {
 	usecolor := term.IsTty(os.Stderr.Fd()) && os.Getenv("TERM") != "dumb"
@@ -103,14 +106,26 @@ func init() {
 	if usecolor {
 		output = colorable.NewColorableStderr()
 	}
-	glogger = log.NewGlogHandler(log.StreamHandler(output, log.TerminalFormat(usecolor)))
+	ostream = log.StreamHandler(output, log.TerminalFormat(usecolor))
+	glogger = log.NewGlogHandler(ostream)
 }
 
 // Setup initializes profiling and logging based on the CLI flags.
 // It should be called as early as possible in the program.
-func Setup(ctx *cli.Context) error {
+func Setup(ctx *cli.Context, logdir string) error {
 	// logging
 	log.PrintOrigins(ctx.GlobalBool(debugFlag.Name))
+	if logdir != "" {
+		rfh, err := log.RotatingFileHandler(
+			logdir,
+			262144,
+			log.JSONFormatOrderedEx(false, true),
+		)
+		if err != nil {
+			return err
+		}
+		glogger.SetHandler(log.MultiHandler(ostream, rfh))
+	}
 	glogger.Verbosity(log.Lvl(ctx.GlobalInt(verbosityFlag.Name)))
 	glogger.Vmodule(ctx.GlobalString(vmoduleFlag.Name))
 	glogger.BacktraceAt(ctx.GlobalString(backtraceAtFlag.Name))

log/format.go

@@ -77,11 +77,11 @@ type TerminalStringer interface {
 // a terminal with color-coded level output and terser human friendly timestamp.
 // This format should only be used for interactive programs or while developing.
 //
-//     [TIME] [LEVEL] MESAGE key=value key=value ...
+//     [LEVEL] [TIME] MESAGE key=value key=value ...
 //
 // Example:
 //
-//     [May 16 20:58:45] [DBUG] remove route ns=haproxy addr=127.0.0.1:50002
+//     [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002
 //
 func TerminalFormat(usecolor bool) Format {
 	return FormatFunc(func(r *Record) []byte {
@@ -202,6 +202,48 @@ func JSONFormat() Format {
 	return JSONFormatEx(false, true)
 }
 
+// JSONFormatOrderedEx formats log records as JSON arrays. If pretty is true,
+// records will be pretty-printed. If lineSeparated is true, records
+// will be logged with a new line between each record.
+func JSONFormatOrderedEx(pretty, lineSeparated bool) Format {
+	jsonMarshal := json.Marshal
+	if pretty {
+		jsonMarshal = func(v interface{}) ([]byte, error) {
+			return json.MarshalIndent(v, "", "    ")
+		}
+	}
+	return FormatFunc(func(r *Record) []byte {
+		props := make(map[string]interface{})
+
+		props[r.KeyNames.Time] = r.Time
+		props[r.KeyNames.Lvl] = r.Lvl.String()
+		props[r.KeyNames.Msg] = r.Msg
+
+		ctx := make([]string, len(r.Ctx))
+		for i := 0; i < len(r.Ctx); i += 2 {
+			k, ok := r.Ctx[i].(string)
+			if !ok {
+				props[errorKey] = fmt.Sprintf("%+v is not a string key,", r.Ctx[i])
+			}
+			ctx[i] = k
+			ctx[i+1] = formatLogfmtValue(r.Ctx[i+1], true)
+		}
+		props[r.KeyNames.Ctx] = ctx
+
+		b, err := jsonMarshal(props)
+		if err != nil {
+			b, _ = jsonMarshal(map[string]string{
+				errorKey: err.Error(),
+			})
+			return b
+		}
+		if lineSeparated {
+			b = append(b, '\n')
+		}
+		return b
+	})
+}
+
 // JSONFormatEx formats log records as JSON objects. If pretty is true,
 // records will be pretty-printed. If lineSeparated is true, records
 // will be logged with a new line between each record.

log/handler.go

@@ -8,6 +8,11 @@ import (
 	"reflect"
 	"sync"
 
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"strings"
+
 	"github.com/go-stack/stack"
 )
 
@@ -70,6 +75,111 @@ func FileHandler(path string, fmtr Format) (Handler, error) {
 	return closingHandler{f, StreamHandler(f, fmtr)}, nil
 }
 
+// countingWriter wraps a WriteCloser object in order to count the written bytes.
+type countingWriter struct {
+	w     io.WriteCloser // the wrapped object
+	count uint           // number of bytes written
+}
+
+// Write increments the byte counter by the number of bytes written.
+// Implements the WriteCloser interface.
+func (w *countingWriter) Write(p []byte) (n int, err error) {
+	n, err = w.w.Write(p)
+	w.count += uint(n)
+	return n, err
+}
+
+// Close implements the WriteCloser interface.
+func (w *countingWriter) Close() error {
+	return w.w.Close()
+}
+
+// prepFile opens the log file at the given path, and cuts off the invalid part
+// from the end, because the previous execution could have been finished by interruption.
+// Assumes that every line ended by '\n' contains a valid log record.
+func prepFile(path string) (*countingWriter, error) {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0600)
+	if err != nil {
+		return nil, err
+	}
+	_, err = f.Seek(-1, io.SeekEnd)
+	if err != nil {
+		return nil, err
+	}
+	buf := make([]byte, 1)
+	var cut int64
+	for {
+		if _, err := f.Read(buf); err != nil {
+			return nil, err
+		}
+		if buf[0] == '\n' {
+			break
+		}
+		if _, err = f.Seek(-2, io.SeekCurrent); err != nil {
+			return nil, err
+		}
+		cut++
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	ns := fi.Size() - cut
+	if err = f.Truncate(ns); err != nil {
+		return nil, err
+	}
+	return &countingWriter{w: f, count: uint(ns)}, nil
+}
+
+// RotatingFileHandler returns a handler which writes log records to file chunks
+// at the given path. When a file's size reaches the limit, the handler creates
+// a new file named after the timestamp of the first log record it will contain.
+func RotatingFileHandler(path string, limit uint, formatter Format) (Handler, error) {
+	if err := os.MkdirAll(path, 0700); err != nil {
+		return nil, err
+	}
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+	re := regexp.MustCompile(`\.log$`)
+	last := len(files) - 1
+	for last >= 0 && (!files[last].Mode().IsRegular() || !re.MatchString(files[last].Name())) {
+		last--
+	}
+	var counter *countingWriter
+	if last >= 0 && files[last].Size() < int64(limit) {
+		// Open the last file, and continue to write into it until it's size reaches the limit.
+		if counter, err = prepFile(filepath.Join(path, files[last].Name())); err != nil {
+			return nil, err
+		}
+	}
+	if counter == nil {
+		counter = new(countingWriter)
+	}
+	h := StreamHandler(counter, formatter)
+
+	return FuncHandler(func(r *Record) error {
+		if counter.count > limit {
+			counter.Close()
+			counter.w = nil
+		}
+		if counter.w == nil {
+			f, err := os.OpenFile(
+				filepath.Join(path, fmt.Sprintf("%s.log", strings.Replace(r.Time.Format("060102150405.00"), ".", "", 1))),
+				os.O_CREATE|os.O_APPEND|os.O_WRONLY,
+				0600,
+			)
+			if err != nil {
+				return err
+			}
+			counter.w = f
+			counter.count = 0
+		}
+		return h.Log(r)
+	}), nil
+}
+
 // NetHandler opens a socket to the given address and writes records
 // over the connection.
 func NetHandler(network, addr string, fmtr Format) (Handler, error) {

log/handler_glog.go

@@ -57,6 +57,11 @@ func NewGlogHandler(h Handler) *GlogHandler {
 	}
 }
 
+// SetHandler updates the handler to write records to the specified sub-handler.
+func (h *GlogHandler) SetHandler(nh Handler) {
+	h.origin = nh
+}
+
 // pattern contains a filter for the Vmodule option, holding a verbosity level
 // and a file pattern to match.
 type pattern struct {

log/logger.go

@@ -11,6 +11,7 @@ import (
 const timeKey = "t"
 const lvlKey = "lvl"
 const msgKey = "msg"
+const ctxKey = "ctx"
 const errorKey = "LOG15_ERROR"
 const skipLevel = 2
 
@@ -101,6 +102,7 @@ type RecordKeyNames struct {
 	Time string
 	Msg  string
 	Lvl  string
+	Ctx  string
 }
 
 // A Logger writes key/value pairs to a Handler
@@ -139,6 +141,7 @@ func (l *logger) write(msg string, lvl Lvl, ctx []interface{}, skip int) {
 			Time: timeKey,
 			Msg:  msgKey,
 			Lvl:  lvlKey,
+			Ctx:  ctxKey,
 		},
 	})
 }

metrics/metrics.go

@@ -58,11 +58,14 @@ func CollectProcessMetrics(refresh time.Duration) {
 	memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
 
 	var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
+	var diskReadBytesCounter, diskWriteBytesCounter Counter
 	if err := ReadDiskStats(diskstats[0]); err == nil {
 		diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
 		diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
+		diskReadBytesCounter = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry)
 		diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
 		diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
+		diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry)
 	} else {
 		log.Debug("Failed to read disk metrics", "err", err)
 	}
@@ -82,6 +85,9 @@ func CollectProcessMetrics(refresh time.Duration) {
 			diskReadBytes.Mark(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
 			diskWrites.Mark(diskstats[location1].WriteCount - diskstats[location2].WriteCount)
 			diskWriteBytes.Mark(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
+
+			diskReadBytesCounter.Inc(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
+			diskWriteBytesCounter.Inc(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
 		}
 		time.Sleep(refresh)
 	}

node/config.go

@@ -179,7 +179,7 @@ func (c *Config) NodeDB() string {
 	if c.DataDir == "" {
 		return "" // ephemeral
 	}
-	return c.resolvePath(datadirNodeDatabase)
+	return c.ResolvePath(datadirNodeDatabase)
 }
 
 // DefaultIPCEndpoint returns the IPC path used by default.
@@ -262,8 +262,8 @@ var isOldGethResource = map[string]bool{
 	"trusted-nodes.json": true,
 }
 
-// resolvePath resolves path in the instance directory.
+// ResolvePath resolves path in the instance directory.
-func (c *Config) resolvePath(path string) string {
+func (c *Config) ResolvePath(path string) string {
 	if filepath.IsAbs(path) {
 		return path
 	}
@@ -309,7 +309,7 @@ func (c *Config) NodeKey() *ecdsa.PrivateKey {
 		return key
 	}
 
-	keyfile := c.resolvePath(datadirPrivateKey)
+	keyfile := c.ResolvePath(datadirPrivateKey)
 	if key, err := crypto.LoadECDSA(keyfile); err == nil {
 		return key
 	}
@@ -332,12 +332,12 @@ func (c *Config) NodeKey() *ecdsa.PrivateKey {
 
 // StaticNodes returns a list of node enode URLs configured as static nodes.
 func (c *Config) StaticNodes() []*discover.Node {
-	return c.parsePersistentNodes(c.resolvePath(datadirStaticNodes))
+	return c.parsePersistentNodes(c.ResolvePath(datadirStaticNodes))
 }
 
 // TrustedNodes returns a list of node enode URLs configured as trusted nodes.
 func (c *Config) TrustedNodes() []*discover.Node {
-	return c.parsePersistentNodes(c.resolvePath(datadirTrustedNodes))
+	return c.parsePersistentNodes(c.ResolvePath(datadirTrustedNodes))
 }
 
 // parsePersistentNodes parses a list of discovery node URLs loaded from a .json

node/node.go

@@ -570,12 +570,12 @@ func (n *Node) OpenDatabase(name string, cache, handles int) (ethdb.Database, er
 	if n.config.DataDir == "" {
 		return ethdb.NewMemDatabase(), nil
 	}
-	return ethdb.NewLDBDatabase(n.config.resolvePath(name), cache, handles)
+	return ethdb.NewLDBDatabase(n.config.ResolvePath(name), cache, handles)
 }
 
 // ResolvePath returns the absolute path of a resource in the instance directory.
 func (n *Node) ResolvePath(x string) string {
-	return n.config.resolvePath(x)
+	return n.config.ResolvePath(x)
 }
 
 // apis returns the collection of RPC descriptors this node offers.

node/service.go

@@ -43,7 +43,7 @@ func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (et
 	if ctx.config.DataDir == "" {
 		return ethdb.NewMemDatabase(), nil
 	}
-	db, err := ethdb.NewLDBDatabase(ctx.config.resolvePath(name), cache, handles)
+	db, err := ethdb.NewLDBDatabase(ctx.config.ResolvePath(name), cache, handles)
 	if err != nil {
 		return nil, err
 	}
@@ -54,7 +54,7 @@ func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (et
 // and if the user actually uses persistent storage. It will return an empty string
 // for emphemeral storage and the user's own input for absolute paths.
 func (ctx *ServiceContext) ResolvePath(path string) string {
-	return ctx.config.resolvePath(path)
+	return ctx.config.ResolvePath(path)
 }
 
 // Service retrieves a currently running service registered of a specific type.

rpc/client.go

@@ -304,7 +304,7 @@ func (c *Client) CallContext(ctx context.Context, result interface{}, method str
 		return err
 	}
 
-	// dispatch has accepted the request and will close the channel it when it quits.
+	// dispatch has accepted the request and will close the channel when it quits.
 	switch resp, err := op.wait(ctx); {
 	case err != nil:
 		return err

swarm/api/api.go

@@ -17,6 +17,7 @@
 package api
 
 import (
+	"archive/tar"
 	"context"
 	"fmt"
 	"io"
@@ -48,6 +49,16 @@ var (
 	apiGetCount            = metrics.NewRegisteredCounter("api.get.count", nil)
 	apiGetNotFound         = metrics.NewRegisteredCounter("api.get.notfound", nil)
 	apiGetHTTP300          = metrics.NewRegisteredCounter("api.get.http.300", nil)
+	apiManifestUpdateCount = metrics.NewRegisteredCounter("api.manifestupdate.count", nil)
+	apiManifestUpdateFail  = metrics.NewRegisteredCounter("api.manifestupdate.fail", nil)
+	apiManifestListCount   = metrics.NewRegisteredCounter("api.manifestlist.count", nil)
+	apiManifestListFail    = metrics.NewRegisteredCounter("api.manifestlist.fail", nil)
+	apiDeleteCount         = metrics.NewRegisteredCounter("api.delete.count", nil)
+	apiDeleteFail          = metrics.NewRegisteredCounter("api.delete.fail", nil)
+	apiGetTarCount         = metrics.NewRegisteredCounter("api.gettar.count", nil)
+	apiGetTarFail          = metrics.NewRegisteredCounter("api.gettar.fail", nil)
+	apiUploadTarCount      = metrics.NewRegisteredCounter("api.uploadtar.count", nil)
+	apiUploadTarFail       = metrics.NewRegisteredCounter("api.uploadtar.fail", nil)
 	apiModifyCount         = metrics.NewRegisteredCounter("api.modify.count", nil)
 	apiModifyFail          = metrics.NewRegisteredCounter("api.modify.fail", nil)
 	apiAddFileCount        = metrics.NewRegisteredCounter("api.addfile.count", nil)
@@ -227,28 +238,28 @@ func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Han
 }
 
 // Upload to be used only in TEST
-func (a *API) Upload(uploadDir, index string, toEncrypt bool) (hash string, err error) {
+func (a *API) Upload(ctx context.Context, uploadDir, index string, toEncrypt bool) (hash string, err error) {
 	fs := NewFileSystem(a)
 	hash, err = fs.Upload(uploadDir, index, toEncrypt)
 	return hash, err
 }
 
 // Retrieve FileStore reader API
-func (a *API) Retrieve(addr storage.Address) (reader storage.LazySectionReader, isEncrypted bool) {
+func (a *API) Retrieve(ctx context.Context, addr storage.Address) (reader storage.LazySectionReader, isEncrypted bool) {
-	return a.fileStore.Retrieve(addr)
+	return a.fileStore.Retrieve(ctx, addr)
 }
 
 // Store wraps the Store API call of the embedded FileStore
-func (a *API) Store(data io.Reader, size int64, toEncrypt bool) (addr storage.Address, wait func(), err error) {
+func (a *API) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr storage.Address, wait func(ctx context.Context) error, err error) {
 	log.Debug("api.store", "size", size)
-	return a.fileStore.Store(data, size, toEncrypt)
+	return a.fileStore.Store(ctx, data, size, toEncrypt)
 }
 
 // ErrResolve is returned when an URI cannot be resolved from ENS.
 type ErrResolve error
 
 // Resolve resolves a URI to an Address using the MultiResolver.
-func (a *API) Resolve(uri *URI) (storage.Address, error) {
+func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) {
 	apiResolveCount.Inc(1)
 	log.Trace("resolving", "uri", uri.Addr)
 
@@ -286,34 +297,37 @@ func (a *API) Resolve(uri *URI) (storage.Address, error) {
 }
 
 // Put provides singleton manifest creation on top of FileStore store
-func (a *API) Put(content, contentType string, toEncrypt bool) (k storage.Address, wait func(), err error) {
+func (a *API) Put(ctx context.Context, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) {
 	apiPutCount.Inc(1)
 	r := strings.NewReader(content)
-	key, waitContent, err := a.fileStore.Store(r, int64(len(content)), toEncrypt)
+	key, waitContent, err := a.fileStore.Store(ctx, r, int64(len(content)), toEncrypt)
 	if err != nil {
 		apiPutFail.Inc(1)
 		return nil, nil, err
 	}
 	manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
 	r = strings.NewReader(manifest)
-	key, waitManifest, err := a.fileStore.Store(r, int64(len(manifest)), toEncrypt)
+	key, waitManifest, err := a.fileStore.Store(ctx, r, int64(len(manifest)), toEncrypt)
 	if err != nil {
 		apiPutFail.Inc(1)
 		return nil, nil, err
 	}
-	return key, func() {
+	return key, func(ctx context.Context) error {
-		waitContent()
+		err := waitContent(ctx)
-		waitManifest()
+		if err != nil {
+			return err
+		}
+		return waitManifest(ctx)
 	}, nil
 }
 
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve basePath to content using FileStore retrieve
 // it returns a section reader, mimeType, status, the key of the actual content and an error
-func (a *API) Get(manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) {
+func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) {
 	log.Debug("api.get", "key", manifestAddr, "path", path)
 	apiGetCount.Inc(1)
-	trie, err := loadManifest(a.fileStore, manifestAddr, nil)
+	trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil)
 	if err != nil {
 		apiGetNotFound.Inc(1)
 		status = http.StatusNotFound
@@ -375,7 +389,7 @@ func (a *API) Get(manifestAddr storage.Address, path string) (reader storage.Laz
 				log.Trace("resource is multihash", "key", manifestAddr)
 
 				// get the manifest the multihash digest points to
-				trie, err := loadManifest(a.fileStore, manifestAddr, nil)
+				trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil)
 				if err != nil {
 					apiGetNotFound.Inc(1)
 					status = http.StatusNotFound
@@ -410,7 +424,7 @@ func (a *API) Get(manifestAddr storage.Address, path string) (reader storage.Laz
 		}
 		mimeType = entry.ContentType
 		log.Debug("content lookup key", "key", contentAddr, "mimetype", mimeType)
-		reader, _ = a.fileStore.Retrieve(contentAddr)
+		reader, _ = a.fileStore.Retrieve(ctx, contentAddr)
 	} else {
 		// no entry found
 		status = http.StatusNotFound
@@ -421,11 +435,190 @@ func (a *API) Get(manifestAddr storage.Address, path string) (reader storage.Laz
 	return
 }
 
+func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Address, error) {
+	apiDeleteCount.Inc(1)
+	uri, err := Parse("bzz:/" + addr)
+	if err != nil {
+		apiDeleteFail.Inc(1)
+		return nil, err
+	}
+	key, err := a.Resolve(ctx, uri)
+
+	if err != nil {
+		return nil, err
+	}
+	newKey, err := a.UpdateManifest(ctx, key, func(mw *ManifestWriter) error {
+		log.Debug(fmt.Sprintf("removing %s from manifest %s", path, key.Log()))
+		return mw.RemoveEntry(path)
+	})
+	if err != nil {
+		apiDeleteFail.Inc(1)
+		return nil, err
+	}
+
+	return newKey, nil
+}
+
+// GetDirectoryTar fetches a requested directory as a tarstream
+// it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser
+func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) {
+	apiGetTarCount.Inc(1)
+	addr, err := a.Resolve(ctx, uri)
+	if err != nil {
+		return nil, err
+	}
+	walker, err := a.NewManifestWalker(ctx, addr, nil)
+	if err != nil {
+		apiGetTarFail.Inc(1)
+		return nil, err
+	}
+
+	piper, pipew := io.Pipe()
+
+	tw := tar.NewWriter(pipew)
+
+	go func() {
+		err := walker.Walk(func(entry *ManifestEntry) error {
+			// ignore manifests (walk will recurse into them)
+			if entry.ContentType == ManifestType {
+				return nil
+			}
+
+			// retrieve the entry's key and size
+			reader, _ := a.Retrieve(ctx, storage.Address(common.Hex2Bytes(entry.Hash)))
+			size, err := reader.Size(nil)
+			if err != nil {
+				return err
+			}
+
+			// write a tar header for the entry
+			hdr := &tar.Header{
+				Name:    entry.Path,
+				Mode:    entry.Mode,
+				Size:    size,
+				ModTime: entry.ModTime,
+				Xattrs: map[string]string{
+					"user.swarm.content-type": entry.ContentType,
+				},
+			}
+
+			if err := tw.WriteHeader(hdr); err != nil {
+				return err
+			}
+
+			// copy the file into the tar stream
+			n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
+			if err != nil {
+				return err
+			} else if n != size {
+				return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
+			}
+
+			return nil
+		})
+		if err != nil {
+			apiGetTarFail.Inc(1)
+			pipew.CloseWithError(err)
+		} else {
+			pipew.Close()
+		}
+	}()
+
+	return piper, nil
+}
+
+// GetManifestList lists the manifest entries for the specified address and prefix
+// and returns it as a ManifestList
+func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) {
+	apiManifestListCount.Inc(1)
+	walker, err := a.NewManifestWalker(ctx, addr, nil)
+	if err != nil {
+		apiManifestListFail.Inc(1)
+		return ManifestList{}, err
+	}
+
+	err = walker.Walk(func(entry *ManifestEntry) error {
+		// handle non-manifest files
+		if entry.ContentType != ManifestType {
+			// ignore the file if it doesn't have the specified prefix
+			if !strings.HasPrefix(entry.Path, prefix) {
+				return nil
+			}
+
+			// if the path after the prefix contains a slash, add a
+			// common prefix to the list, otherwise add the entry
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return nil
+			}
+			if entry.Path == "" {
+				entry.Path = "/"
+			}
+			list.Entries = append(list.Entries, entry)
+			return nil
+		}
+
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by returning nil and
+		// continuing the walk
+		if strings.HasPrefix(prefix, entry.Path) {
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a slash, add a common prefix
+		// to the list and skip the manifest, otherwise recurse into
+		// the manifest by returning nil and continuing the walk
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return ErrSkipManifest
+			}
+			return nil
+		}
+
+		// the manifest neither has the prefix or needs recursing in to
+		// so just skip it
+		return ErrSkipManifest
+	})
+
+	if err != nil {
+		apiManifestListFail.Inc(1)
+		return ManifestList{}, err
+	}
+
+	return list, nil
+}
+
+func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update func(mw *ManifestWriter) error) (storage.Address, error) {
+	apiManifestUpdateCount.Inc(1)
+	mw, err := a.NewManifestWriter(ctx, addr, nil)
+	if err != nil {
+		apiManifestUpdateFail.Inc(1)
+		return nil, err
+	}
+
+	if err := update(mw); err != nil {
+		apiManifestUpdateFail.Inc(1)
+		return nil, err
+	}
+
+	addr, err = mw.Store()
+	if err != nil {
+		apiManifestUpdateFail.Inc(1)
+		return nil, err
+	}
+	log.Debug(fmt.Sprintf("generated manifest %s", addr))
+	return addr, nil
+}
+
 // Modify loads manifest and checks the content hash before recalculating and storing the manifest.
-func (a *API) Modify(addr storage.Address, path, contentHash, contentType string) (storage.Address, error) {
+func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) {
 	apiModifyCount.Inc(1)
 	quitC := make(chan bool)
-	trie, err := loadManifest(a.fileStore, addr, quitC)
+	trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
 	if err != nil {
 		apiModifyFail.Inc(1)
 		return nil, err
@@ -449,7 +642,7 @@ func (a *API) Modify(addr storage.Address, path, contentHash, contentType string
 }
 
 // AddFile creates a new manifest entry, adds it to swarm, then adds a file to swarm.
-func (a *API) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Address, string, error) {
+func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []byte, nameresolver bool) (storage.Address, string, error) {
 	apiAddFileCount.Inc(1)
 
 	uri, err := Parse("bzz:/" + mhash)
@@ -457,7 +650,7 @@ func (a *API) AddFile(mhash, path, fname string, content []byte, nameresolver bo
 		apiAddFileFail.Inc(1)
 		return nil, "", err
 	}
-	mkey, err := a.Resolve(uri)
+	mkey, err := a.Resolve(ctx, uri)
 	if err != nil {
 		apiAddFileFail.Inc(1)
 		return nil, "", err
@@ -476,13 +669,13 @@ func (a *API) AddFile(mhash, path, fname string, content []byte, nameresolver bo
 		ModTime:     time.Now(),
 	}
 
-	mw, err := a.NewManifestWriter(mkey, nil)
+	mw, err := a.NewManifestWriter(ctx, mkey, nil)
 	if err != nil {
 		apiAddFileFail.Inc(1)
 		return nil, "", err
 	}
 
-	fkey, err := mw.AddEntry(bytes.NewReader(content), entry)
+	fkey, err := mw.AddEntry(ctx, bytes.NewReader(content), entry)
 	if err != nil {
 		apiAddFileFail.Inc(1)
 		return nil, "", err
@@ -496,11 +689,47 @@ func (a *API) AddFile(mhash, path, fname string, content []byte, nameresolver bo
 	}
 
 	return fkey, newMkey.String(), nil
+}
 
+func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) {
+	apiUploadTarCount.Inc(1)
+	var contentKey storage.Address
+	tr := tar.NewReader(bodyReader)
+	defer bodyReader.Close()
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			apiUploadTarFail.Inc(1)
+			return nil, fmt.Errorf("error reading tar stream: %s", err)
+		}
+
+		// only store regular files
+		if !hdr.FileInfo().Mode().IsRegular() {
+			continue
+		}
+
+		// add the entry under the path from the request
+		manifestPath := path.Join(manifestPath, hdr.Name)
+		entry := &ManifestEntry{
+			Path:        manifestPath,
+			ContentType: hdr.Xattrs["user.swarm.content-type"],
+			Mode:        hdr.Mode,
+			Size:        hdr.Size,
+			ModTime:     hdr.ModTime,
+		}
+		contentKey, err = mw.AddEntry(ctx, tr, entry)
+		if err != nil {
+			apiUploadTarFail.Inc(1)
+			return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err)
+		}
+	}
+	return contentKey, nil
 }
 
 // RemoveFile removes a file entry in a manifest.
-func (a *API) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) {
+func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname string, nameresolver bool) (string, error) {
 	apiRmFileCount.Inc(1)
 
 	uri, err := Parse("bzz:/" + mhash)
@@ -508,7 +737,7 @@ func (a *API) RemoveFile(mhash, path, fname string, nameresolver bool) (string,
 		apiRmFileFail.Inc(1)
 		return "", err
 	}
-	mkey, err := a.Resolve(uri)
+	mkey, err := a.Resolve(ctx, uri)
 	if err != nil {
 		apiRmFileFail.Inc(1)
 		return "", err
@@ -519,7 +748,7 @@ func (a *API) RemoveFile(mhash, path, fname string, nameresolver bool) (string,
 		path = path[1:]
 	}
 
-	mw, err := a.NewManifestWriter(mkey, nil)
+	mw, err := a.NewManifestWriter(ctx, mkey, nil)
 	if err != nil {
 		apiRmFileFail.Inc(1)
 		return "", err
@@ -542,7 +771,7 @@ func (a *API) RemoveFile(mhash, path, fname string, nameresolver bool) (string,
 }
 
 // AppendFile removes old manifest, appends file entry to new manifest and adds it to Swarm.
-func (a *API) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldAddr storage.Address, offset int64, addSize int64, nameresolver bool) (storage.Address, string, error) {
+func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existingSize int64, content []byte, oldAddr storage.Address, offset int64, addSize int64, nameresolver bool) (storage.Address, string, error) {
 	apiAppendFileCount.Inc(1)
 
 	buffSize := offset + addSize
@@ -552,7 +781,7 @@ func (a *API) AppendFile(mhash, path, fname string, existingSize int64, content
 
 	buf := make([]byte, buffSize)
 
-	oldReader, _ := a.Retrieve(oldAddr)
+	oldReader, _ := a.Retrieve(ctx, oldAddr)
 	io.ReadAtLeast(oldReader, buf, int(offset))
 
 	newReader := bytes.NewReader(content)
@@ -575,7 +804,7 @@ func (a *API) AppendFile(mhash, path, fname string, existingSize int64, content
 		apiAppendFileFail.Inc(1)
 		return nil, "", err
 	}
-	mkey, err := a.Resolve(uri)
+	mkey, err := a.Resolve(ctx, uri)
 	if err != nil {
 		apiAppendFileFail.Inc(1)
 		return nil, "", err
@@ -586,7 +815,7 @@ func (a *API) AppendFile(mhash, path, fname string, existingSize int64, content
 		path = path[1:]
 	}
 
-	mw, err := a.NewManifestWriter(mkey, nil)
+	mw, err := a.NewManifestWriter(ctx, mkey, nil)
 	if err != nil {
 		apiAppendFileFail.Inc(1)
 		return nil, "", err
@@ -606,7 +835,7 @@ func (a *API) AppendFile(mhash, path, fname string, existingSize int64, content
 		ModTime:     time.Now(),
 	}
 
-	fkey, err := mw.AddEntry(io.Reader(combinedReader), entry)
+	fkey, err := mw.AddEntry(ctx, io.Reader(combinedReader), entry)
 	if err != nil {
 		apiAppendFileFail.Inc(1)
 		return nil, "", err
@@ -620,23 +849,22 @@ func (a *API) AppendFile(mhash, path, fname string, existingSize int64, content
 	}
 
 	return fkey, newMkey.String(), nil
-
 }
 
 // BuildDirectoryTree used by swarmfs_unix
-func (a *API) BuildDirectoryTree(mhash string, nameresolver bool) (addr storage.Address, manifestEntryMap map[string]*manifestTrieEntry, err error) {
+func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver bool) (addr storage.Address, manifestEntryMap map[string]*manifestTrieEntry, err error) {
 
 	uri, err := Parse("bzz:/" + mhash)
 	if err != nil {
 		return nil, nil, err
 	}
-	addr, err = a.Resolve(uri)
+	addr, err = a.Resolve(ctx, uri)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	quitC := make(chan bool)
-	rootTrie, err := loadManifest(a.fileStore, addr, quitC)
+	rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC)
 	if err != nil {
 		return nil, nil, fmt.Errorf("can't load manifest %v: %v", addr.String(), err)
 	}
@@ -725,8 +953,8 @@ func (a *API) ResourceIsValidated() bool {
 }
 
 // ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk.
-func (a *API) ResolveResourceManifest(addr storage.Address) (storage.Address, error) {
+func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) {
-	trie, err := loadManifest(a.fileStore, addr, nil)
+	trie, err := loadManifest(ctx, a.fileStore, addr, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot load resource manifest: %v", err)
 	}

swarm/api/api_test.go

@@ -85,7 +85,7 @@ func expResponse(content string, mimeType string, status int) *Response {
 
 func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse {
 	addr := storage.Address(common.Hex2Bytes(bzzhash))
-	reader, mimeType, status, _, err := api.Get(addr, path)
+	reader, mimeType, status, _, err := api.Get(context.TODO(), addr, path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -109,12 +109,15 @@ func TestApiPut(t *testing.T) {
 	testAPI(t, func(api *API, toEncrypt bool) {
 		content := "hello"
 		exp := expResponse(content, "text/plain", 0)
-		// exp := expResponse([]byte(content), "text/plain", 0)
+		ctx := context.TODO()
-		addr, wait, err := api.Put(content, exp.MimeType, toEncrypt)
+		addr, wait, err := api.Put(ctx, content, exp.MimeType, toEncrypt)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		err = wait(ctx)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		wait()
 		resp := testGet(t, api, addr.Hex(), "")
 		checkResponse(t, resp, exp)
 	})
@@ -226,7 +229,7 @@ func TestAPIResolve(t *testing.T) {
 			if x.immutable {
 				uri.Scheme = "bzz-immutable"
 			}
-			res, err := api.Resolve(uri)
+			res, err := api.Resolve(context.TODO(), uri)
 			if err == nil {
 				if x.expectErr != nil {
 					t.Fatalf("expected error %q, got result %q", x.expectErr, res)

swarm/api/client/client_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 func serverFunc(api *api.API) testutil.TestServer {
-	return swarmhttp.NewServer(api)
+	return swarmhttp.NewServer(api, "")
 }
 
 // TestClientUploadDownloadRaw test uploading and downloading raw data to swarm

swarm/api/filesystem.go

@@ -18,6 +18,7 @@ package api
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -113,12 +114,13 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error
 			if err == nil {
 				stat, _ := f.Stat()
 				var hash storage.Address
-				var wait func()
+				var wait func(context.Context) error
-				hash, wait, err = fs.api.fileStore.Store(f, stat.Size(), toEncrypt)
+				ctx := context.TODO()
+				hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt)
 				if hash != nil {
 					list[i].Hash = hash.Hex()
 				}
-				wait()
+				err = wait(ctx)
 				awg.Done()
 				if err == nil {
 					first512 := make([]byte, 512)
@@ -189,7 +191,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error {
 	if err != nil {
 		return err
 	}
-	addr, err := fs.api.Resolve(uri)
+	addr, err := fs.api.Resolve(context.TODO(), uri)
 	if err != nil {
 		return err
 	}
@@ -200,7 +202,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error {
 	}
 
 	quitC := make(chan bool)
-	trie, err := loadManifest(fs.api.fileStore, addr, quitC)
+	trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC)
 	if err != nil {
 		log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
 		return err
@@ -273,7 +275,7 @@ func retrieveToFile(quitC chan bool, fileStore *storage.FileStore, addr storage.
 	if err != nil {
 		return err
 	}
-	reader, _ := fileStore.Retrieve(addr)
+	reader, _ := fileStore.Retrieve(context.TODO(), addr)
 	writer := bufio.NewWriter(f)
 	size, err := reader.Size(quitC)
 	if err != nil {

swarm/api/filesystem_test.go

@@ -18,6 +18,7 @@ package api
 
 import (
 	"bytes"
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -63,7 +64,7 @@ func TestApiDirUpload0(t *testing.T) {
 		checkResponse(t, resp, exp)
 
 		addr := storage.Address(common.Hex2Bytes(bzzhash))
-		_, _, _, _, err = api.Get(addr, "")
+		_, _, _, _, err = api.Get(context.TODO(), addr, "")
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
@@ -95,7 +96,7 @@ func TestApiDirUploadModify(t *testing.T) {
 		}
 
 		addr := storage.Address(common.Hex2Bytes(bzzhash))
-		addr, err = api.Modify(addr, "index.html", "", "")
+		addr, err = api.Modify(context.TODO(), addr, "index.html", "", "")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -105,18 +106,23 @@ func TestApiDirUploadModify(t *testing.T) {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index)), toEncrypt)
+		ctx := context.TODO()
-		wait()
+		hash, wait, err := api.Store(ctx, bytes.NewReader(index), int64(len(index)), toEncrypt)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		addr, err = api.Modify(addr, "index2.html", hash.Hex(), "text/html; charset=utf-8")
+		err = wait(ctx)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		addr, err = api.Modify(addr, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
+		addr, err = api.Modify(context.TODO(), addr, "index2.html", hash.Hex(), "text/html; charset=utf-8")
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		addr, err = api.Modify(context.TODO(), addr, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -137,7 +143,7 @@ func TestApiDirUploadModify(t *testing.T) {
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, _, err = api.Get(addr, "")
+		_, _, _, _, err = api.Get(context.TODO(), addr, "")
 		if err == nil {
 			t.Errorf("expected error: %v", err)
 		}

swarm/api/http/error.go

@@ -147,6 +147,14 @@ func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
 	switch code {
 	case http.StatusInternalServerError:
 		log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code)
+	case http.StatusMultipleChoices:
+		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
+		listURI := api.URI{
+			Scheme: "bzz-list",
+			Addr:   req.uri.Addr,
+			Path:   req.uri.Path,
+		}
+		additionalMessage = fmt.Sprintf(`<a href="/%s">multiple choices</a>`, listURI.String())
 	default:
 		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
 	}

swarm/api/http/error_test.go

@@ -29,7 +29,6 @@ import (
 )
 
 func TestError(t *testing.T) {
-
 	srv := testutil.NewTestSwarmServer(t, serverFunc)
 	defer srv.Close()
 

swarm/api/http/server.go

@@ -20,7 +20,6 @@ A simple http server interface to Swarm
 package http
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
 	"encoding/json"
@@ -67,17 +66,171 @@ var (
 	getFileCount    = metrics.NewRegisteredCounter("api.http.get.file.count", nil)
 	getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil)
 	getFileFail     = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
-	getFilesCount   = metrics.NewRegisteredCounter("api.http.get.files.count", nil)
-	getFilesFail    = metrics.NewRegisteredCounter("api.http.get.files.fail", nil)
 	getListCount    = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
 	getListFail     = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
 )
 
-// ServerConfig is the basic configuration needed for the HTTP server and also
+func NewServer(api *api.API, corsString string) *Server {
-// includes CORS settings.
+	var allowedOrigins []string
-type ServerConfig struct {
+	for _, domain := range strings.Split(corsString, ",") {
-	Addr       string
+		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
-	CorsString string
+	}
+	c := cors.New(cors.Options{
+		AllowedOrigins: allowedOrigins,
+		AllowedMethods: []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodPatch, http.MethodPut},
+		MaxAge:         600,
+		AllowedHeaders: []string{"*"},
+	})
+
+	mux := http.NewServeMux()
+	server := &Server{api: api}
+	mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz))
+	mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw))
+	mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable))
+	mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash))
+	mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList))
+	mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource))
+
+	mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths))
+	mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths))
+	mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths))
+
+	server.Handler = c.Handler(mux)
+	return server
+}
+func (s *Server) ListenAndServe(addr string) error {
+	return http.ListenAndServe(addr, s)
+}
+func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		if r.RequestURI == "/" {
+			if strings.Contains(r.Header.Get("Accept"), "text/html") {
+				err := landingPageTemplate.Execute(w, nil)
+				if err != nil {
+					log.Error(fmt.Sprintf("error rendering landing page: %s", err))
+				}
+				return
+			}
+			if strings.Contains(r.Header.Get("Accept"), "application/json") {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				json.NewEncoder(w).Encode("Welcome to Swarm!")
+				return
+			}
+		}
+
+		if r.URL.Path == "/robots.txt" {
+			w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+			fmt.Fprintf(w, "User-agent: *\nDisallow: /")
+			return
+		}
+		Respond(w, r, "Bad Request", http.StatusBadRequest)
+	default:
+		Respond(w, r, "Not Found", http.StatusNotFound)
+	}
+}
+func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetBzz")
+		if r.Header.Get("Accept") == "application/x-tar" {
+			reader, err := s.api.GetDirectoryTar(r.Context(), r.uri)
+			if err != nil {
+				Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
+			}
+			defer reader.Close()
+
+			w.Header().Set("Content-Type", "application/x-tar")
+			w.WriteHeader(http.StatusOK)
+			io.Copy(w, reader)
+			return
+		}
+		s.HandleGetFile(w, r)
+	case http.MethodPost:
+		log.Debug("handlePostFiles")
+		s.HandlePostFiles(w, r)
+	case http.MethodDelete:
+		log.Debug("handleBzzDelete")
+		s.HandleDelete(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetRaw")
+		s.HandleGet(w, r)
+	case http.MethodPost:
+		log.Debug("handlePostRaw")
+		s.HandlePostRaw(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetHash")
+		s.HandleGetList(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetHash")
+		s.HandleGet(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetHash")
+		s.HandleGetList(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) {
+	switch r.Method {
+	case http.MethodGet:
+		log.Debug("handleGetResource")
+		s.HandleGetResource(w, r)
+	case http.MethodPost:
+		log.Debug("handlePostResource")
+		s.HandlePostResource(w, r)
+	default:
+		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
+		req := &Request{Request: *r, ruid: uuid.New()[:8]}
+		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
+		log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
+
+		// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
+		w := newLoggingResponseWriter(rw)
+		if parseBzzUri {
+			uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+			if err != nil {
+				Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
+				return
+			}
+			req.uri = uri
+
+			log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
+		}
+
+		h(w, req) // call original
+		log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
+	})
 }
 
 // browser API for registering bzz url scheme handlers:
@@ -85,28 +238,13 @@ type ServerConfig struct {
 // electron (chromium) api for registering bzz url scheme handlers:
 // https://github.com/atom/electron/blob/master/docs/api/protocol.md
 
-// starts up http server
+// browser API for registering bzz url scheme handlers:
-func StartHTTPServer(api *api.API, config *ServerConfig) {
+// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
-	var allowedOrigins []string
+// electron (chromium) api for registering bzz url scheme handlers:
-	for _, domain := range strings.Split(config.CorsString, ",") {
+// https://github.com/atom/electron/blob/master/docs/api/protocol.md
-		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
-	}
-	c := cors.New(cors.Options{
-		AllowedOrigins: allowedOrigins,
-		AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"},
-		MaxAge:         600,
-		AllowedHeaders: []string{"*"},
-	})
-	hdlr := c.Handler(NewServer(api))
-
-	go http.ListenAndServe(config.Addr, hdlr)
-}
-
-func NewServer(api *api.API) *Server {
-	return &Server{api}
-}
 
 type Server struct {
+	http.Handler
 	api *api.API
 }
 
@@ -147,7 +285,8 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 		Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest)
 		return
 	}
-	addr, _, err := s.api.Store(r.Body, r.ContentLength, toEncrypt)
+
+	addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
 	if err != nil {
 		postRawFail.Inc(1)
 		Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -184,7 +323,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 
 	var addr storage.Address
 	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
-		addr, err = s.api.Resolve(r.uri)
+		addr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			postFilesFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
@@ -192,7 +331,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 		}
 		log.Debug("resolved key", "ruid", r.ruid, "key", addr)
 	} else {
-		addr, err = s.api.NewManifest(toEncrypt)
+		addr, err = s.api.NewManifest(r.Context(), toEncrypt)
 		if err != nil {
 			postFilesFail.Inc(1)
 			Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -201,12 +340,16 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 		log.Debug("new manifest", "ruid", r.ruid, "key", addr)
 	}
 
-	newAddr, err := s.updateManifest(addr, func(mw *api.ManifestWriter) error {
+	newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
 		switch contentType {
 
 		case "application/x-tar":
-			return s.handleTarUpload(r, mw)
+			_, err := s.handleTarUpload(r, mw)
-
+			if err != nil {
+				Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
+				return err
+			}
+			return nil
 		case "multipart/form-data":
 			return s.handleMultipartUpload(r, params["boundary"], mw)
 
@@ -227,38 +370,14 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 	fmt.Fprint(w, newAddr)
 }
 
-func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
+func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) {
-	log.Debug("handle.tar.upload", "ruid", req.ruid)
+	log.Debug("handle.tar.upload", "ruid", r.ruid)
-	tr := tar.NewReader(req.Body)
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			return nil
-		} else if err != nil {
-			return fmt.Errorf("error reading tar stream: %s", err)
-		}
 
-		// only store regular files
+	key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw)
-		if !hdr.FileInfo().Mode().IsRegular() {
-			continue
-		}
-
-		// add the entry under the path from the request
-		path := path.Join(req.uri.Path, hdr.Name)
-		entry := &api.ManifestEntry{
-			Path:        path,
-			ContentType: hdr.Xattrs["user.swarm.content-type"],
-			Mode:        hdr.Mode,
-			Size:        hdr.Size,
-			ModTime:     hdr.ModTime,
-		}
-		log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
-		contentKey, err := mw.AddEntry(tr, entry)
 	if err != nil {
-			return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
+		return nil, err
-		}
-		log.Debug("stored content", "ruid", req.ruid, "key", contentKey)
 	}
+	return key, nil
 }
 
 func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
@@ -311,7 +430,7 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma
 			ModTime:     time.Now(),
 		}
 		log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
-		contentKey, err := mw.AddEntry(reader, entry)
+		contentKey, err := mw.AddEntry(req.Context(), reader, entry)
 		if err != nil {
 			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
 		}
@@ -321,7 +440,7 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma
 
 func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
 	log.Debug("handle.direct.upload", "ruid", req.ruid)
-	key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
+	key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{
 		Path:        req.uri.Path,
 		ContentType: req.Header.Get("Content-Type"),
 		Mode:        0644,
@@ -340,22 +459,11 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error
 // text/plain response
 func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.delete", "ruid", r.ruid)
-
 	deleteCount.Inc(1)
-	key, err := s.api.Resolve(r.uri)
+	newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path)
 	if err != nil {
 		deleteFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
+		Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
-		return
-	}
-
-	newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
-		log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log()), "ruid", r.ruid)
-		return mw.RemoveEntry(r.uri.Path)
-	})
-	if err != nil {
-		deleteFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot update manifest: %s", err), http.StatusInternalServerError)
 		return
 	}
 
@@ -428,7 +536,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// we create a manifest so we can retrieve the resource with bzz:// later
 		// this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource
 		// root chunk
-		m, err := s.api.NewResourceManifest(addr.Hex())
+		m, err := s.api.NewResourceManifest(r.Context(), addr.Hex())
 		if err != nil {
 			Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
 			return
@@ -448,7 +556,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// that means that we retrieve the manifest and inspect its Hash member.
 		manifestAddr := r.uri.Address()
 		if manifestAddr == nil {
-			manifestAddr, err = s.api.Resolve(r.uri)
+			manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
 			if err != nil {
 				getFail.Inc(1)
 				Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -459,7 +567,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		}
 
 		// get the root chunk key from the manifest
-		addr, err = s.api.ResolveResourceManifest(manifestAddr)
+		addr, err = s.api.ResolveResourceManifest(r.Context(), manifestAddr)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -518,19 +626,15 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 // bzz-resource://<id>/<n> - get latest update on period n
 // bzz-resource://<id>/<n>/<m> - get update version m of period n
 // <id> = ens name or hash
-func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
-	s.handleGetResource(w, r)
-}
-
 // TODO: Enable pass maxPeriod parameter
-func (s *Server) handleGetResource(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.get.resource", "ruid", r.ruid)
 	var err error
 
 	// resolve the content key.
 	manifestAddr := r.uri.Address()
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -541,7 +645,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request) {
 	}
 
 	// get the root chunk key from the manifest
-	key, err := s.api.ResolveResourceManifest(manifestAddr)
+	key, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr)
 	if err != nil {
 		getFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -629,7 +733,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	var err error
 	addr := r.uri.Address()
 	if addr == nil {
-		addr, err = s.api.Resolve(r.uri)
+		addr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -644,7 +748,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	// if path is set, interpret <key> as a manifest and return the
 	// raw entry at the given path
 	if r.uri.Path != "" {
-		walker, err := s.api.NewManifestWalker(addr, nil)
+		walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
 		if err != nil {
 			getFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
@@ -692,7 +796,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	reader, isEncrypted := s.api.Retrieve(addr)
+	reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
 	if _, err := reader.Size(nil); err != nil {
 		getFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
@@ -718,82 +822,6 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	}
 }
 
-// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
-// header of "application/x-tar" and returns a tar stream of all files
-// contained in the manifest
-func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
-	log.Debug("handle.get.files", "ruid", r.ruid, "uri", r.uri)
-	getFilesCount.Inc(1)
-	if r.uri.Path != "" {
-		getFilesFail.Inc(1)
-		Respond(w, r, "files request cannot contain a path", http.StatusBadRequest)
-		return
-	}
-
-	addr, err := s.api.Resolve(r.uri)
-	if err != nil {
-		getFilesFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
-		return
-	}
-	log.Debug("handle.get.files: resolved", "ruid", r.ruid, "key", addr)
-
-	walker, err := s.api.NewManifestWalker(addr, nil)
-	if err != nil {
-		getFilesFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	tw := tar.NewWriter(w)
-	defer tw.Close()
-	w.Header().Set("Content-Type", "application/x-tar")
-	w.WriteHeader(http.StatusOK)
-
-	err = walker.Walk(func(entry *api.ManifestEntry) error {
-		// ignore manifests (walk will recurse into them)
-		if entry.ContentType == api.ManifestType {
-			return nil
-		}
-
-		// retrieve the entry's key and size
-		reader, isEncrypted := s.api.Retrieve(storage.Address(common.Hex2Bytes(entry.Hash)))
-		size, err := reader.Size(nil)
-		if err != nil {
-			return err
-		}
-		w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
-
-		// write a tar header for the entry
-		hdr := &tar.Header{
-			Name:    entry.Path,
-			Mode:    entry.Mode,
-			Size:    size,
-			ModTime: entry.ModTime,
-			Xattrs: map[string]string{
-				"user.swarm.content-type": entry.ContentType,
-			},
-		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-
-		// copy the file into the tar stream
-		n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
-		if err != nil {
-			return err
-		} else if n != size {
-			return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
-		}
-
-		return nil
-	})
-	if err != nil {
-		getFilesFail.Inc(1)
-		log.Error(fmt.Sprintf("error generating tar stream: %s", err))
-	}
-}
-
 // HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
 // a list of all files contained in <manifest> under <path> grouped into
 // common prefixes using "/" as a delimiter
@@ -806,7 +834,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 		return
 	}
 
-	addr, err := s.api.Resolve(r.uri)
+	addr, err := s.api.Resolve(r.Context(), r.uri)
 	if err != nil {
 		getListFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -814,8 +842,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	}
 	log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr)
 
-	list, err := s.getManifestList(addr, r.uri.Path)
+	list, err := s.api.GetManifestList(r.Context(), addr, r.uri.Path)
-
 	if err != nil {
 		getListFail.Inc(1)
 		Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -845,62 +872,6 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	json.NewEncoder(w).Encode(&list)
 }
 
-func (s *Server) getManifestList(addr storage.Address, prefix string) (list api.ManifestList, err error) {
-	walker, err := s.api.NewManifestWalker(addr, nil)
-	if err != nil {
-		return
-	}
-
-	err = walker.Walk(func(entry *api.ManifestEntry) error {
-		// handle non-manifest files
-		if entry.ContentType != api.ManifestType {
-			// ignore the file if it doesn't have the specified prefix
-			if !strings.HasPrefix(entry.Path, prefix) {
-				return nil
-			}
-
-			// if the path after the prefix contains a slash, add a
-			// common prefix to the list, otherwise add the entry
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			if index := strings.Index(suffix, "/"); index > -1 {
-				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
-				return nil
-			}
-			if entry.Path == "" {
-				entry.Path = "/"
-			}
-			list.Entries = append(list.Entries, entry)
-			return nil
-		}
-
-		// if the manifest's path is a prefix of the specified prefix
-		// then just recurse into the manifest by returning nil and
-		// continuing the walk
-		if strings.HasPrefix(prefix, entry.Path) {
-			return nil
-		}
-
-		// if the manifest's path has the specified prefix, then if the
-		// path after the prefix contains a slash, add a common prefix
-		// to the list and skip the manifest, otherwise recurse into
-		// the manifest by returning nil and continuing the walk
-		if strings.HasPrefix(entry.Path, prefix) {
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			if index := strings.Index(suffix, "/"); index > -1 {
-				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
-				return api.ErrSkipManifest
-			}
-			return nil
-		}
-
-		// the manifest neither has the prefix or needs recursing in to
-		// so just skip it
-		return api.ErrSkipManifest
-	})
-
-	return list, nil
-}
-
 // HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
 // with the content of the file at <path> from the given <manifest>
 func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
@@ -915,7 +886,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	manifestAddr := r.uri.Address()
 
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
 		if err != nil {
 			getFileFail.Inc(1)
 			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
@@ -926,8 +897,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	}
 
 	log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr)
-
+	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path)
-	reader, contentType, status, contentKey, err := s.api.Get(manifestAddr, r.uri.Path)
 
 	etag := common.Bytes2Hex(contentKey)
 	noneMatchEtag := r.Header.Get("If-None-Match")
@@ -954,8 +924,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	//the request results in ambiguous files
 	//e.g. /read with readme.md and readinglist.txt available in manifest
 	if status == http.StatusMultipleChoices {
-		list, err := s.getManifestList(manifestAddr, r.uri.Path)
+		list, err := s.api.GetManifestList(r.Context(), manifestAddr, r.uri.Path)
-
 		if err != nil {
 			getFileFail.Inc(1)
 			Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -1010,123 +979,6 @@ func (b bufferedReadSeeker) Seek(offset int64, whence int) (int64, error) {
 	return b.s.Seek(offset, whence)
 }
 
-func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
-	req := &Request{Request: *r, ruid: uuid.New()[:8]}
-	metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
-	log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
-
-	// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
-	w := newLoggingResponseWriter(rw)
-
-	if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") {
-
-		err := landingPageTemplate.Execute(w, nil)
-		if err != nil {
-			log.Error(fmt.Sprintf("error rendering landing page: %s", err))
-		}
-		return
-	}
-
-	if r.URL.Path == "/robots.txt" {
-		w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
-		fmt.Fprintf(w, "User-agent: *\nDisallow: /")
-		return
-	}
-
-	if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "application/json") {
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusOK)
-		json.NewEncoder(w).Encode("Welcome to Swarm!")
-		return
-	}
-
-	uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
-	if err != nil {
-		Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
-		return
-	}
-
-	req.uri = uri
-
-	log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
-
-	switch r.Method {
-	case "POST":
-		if uri.Raw() {
-			log.Debug("handlePostRaw")
-			s.HandlePostRaw(w, req)
-		} else if uri.Resource() {
-			log.Debug("handlePostResource")
-			s.HandlePostResource(w, req)
-		} else if uri.Immutable() || uri.List() || uri.Hash() {
-			log.Debug("POST not allowed on immutable, list or hash")
-			Respond(w, req, fmt.Sprintf("POST method on scheme %s not allowed", uri.Scheme), http.StatusMethodNotAllowed)
-		} else {
-			log.Debug("handlePostFiles")
-			s.HandlePostFiles(w, req)
-		}
-
-	case "PUT":
-		Respond(w, req, fmt.Sprintf("PUT method to %s not allowed", uri), http.StatusBadRequest)
-		return
-
-	case "DELETE":
-		if uri.Raw() {
-			Respond(w, req, fmt.Sprintf("DELETE method to %s not allowed", uri), http.StatusBadRequest)
-			return
-		}
-		s.HandleDelete(w, req)
-
-	case "GET":
-
-		if uri.Resource() {
-			s.HandleGetResource(w, req)
-			return
-		}
-
-		if uri.Raw() || uri.Hash() {
-			s.HandleGet(w, req)
-			return
-		}
-
-		if uri.List() {
-			s.HandleGetList(w, req)
-			return
-		}
-
-		if r.Header.Get("Accept") == "application/x-tar" {
-			s.HandleGetFiles(w, req)
-			return
-		}
-
-		s.HandleGetFile(w, req)
-
-	default:
-		Respond(w, req, fmt.Sprintf("%s method is not supported", r.Method), http.StatusMethodNotAllowed)
-	}
-
-	log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
-}
-
-func (s *Server) updateManifest(addr storage.Address, update func(mw *api.ManifestWriter) error) (storage.Address, error) {
-	mw, err := s.api.NewManifestWriter(addr, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := update(mw); err != nil {
-		return nil, err
-	}
-
-	addr, err = mw.Store()
-	if err != nil {
-		return nil, err
-	}
-	log.Debug(fmt.Sprintf("generated manifest %s", addr))
-	return addr, nil
-}
-
 type loggingResponseWriter struct {
 	http.ResponseWriter
 	statusCode int

swarm/api/http/server_test.go

@@ -17,17 +17,21 @@
 package http
 
 import (
+	"archive/tar"
 	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -87,7 +91,7 @@ func TestResourcePostMode(t *testing.T) {
 }
 
 func serverFunc(api *api.API) testutil.TestServer {
-	return NewServer(api)
+	return NewServer(api, "")
 }
 
 // test the transparent resolving of multihash resource types with bzz:// scheme
@@ -355,6 +359,11 @@ func TestBzzGetPath(t *testing.T) {
 	testBzzGetPath(true, t)
 }
 
+func TestBzzTar(t *testing.T) {
+	testBzzTar(false, t)
+	testBzzTar(true, t)
+}
+
 func testBzzGetPath(encrypted bool, t *testing.T) {
 	var err error
 
@@ -382,15 +391,19 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
 
 	for i, mf := range testmanifest {
 		reader[i] = bytes.NewReader([]byte(mf))
-		var wait func()
+		var wait func(context.Context) error
-		addr[i], wait, err = srv.FileStore.Store(reader[i], int64(len(mf)), encrypted)
+		ctx := context.TODO()
+		addr[i], wait, err = srv.FileStore.Store(ctx, reader[i], int64(len(mf)), encrypted)
 		for j := i + 1; j < len(testmanifest); j++ {
 			testmanifest[j] = strings.Replace(testmanifest[j], fmt.Sprintf("<key%v>", i), addr[i].Hex(), -1)
 		}
 		if err != nil {
 			t.Fatal(err)
 		}
-		wait()
+		err = wait(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}
 
 	rootRef := addr[2].Hex()
@@ -587,6 +600,122 @@ func testBzzGetPath(encrypted bool, t *testing.T) {
 	}
 }
 
+func testBzzTar(encrypted bool, t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t, serverFunc)
+	defer srv.Close()
+	fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"}
+	fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"}
+
+	buf := &bytes.Buffer{}
+	tw := tar.NewWriter(buf)
+	defer tw.Close()
+
+	for i, v := range fileNames {
+		size := int64(len(fileContents[i]))
+		hdr := &tar.Header{
+			Name:    v,
+			Mode:    0644,
+			Size:    size,
+			ModTime: time.Now(),
+			Xattrs: map[string]string{
+				"user.swarm.content-type": "text/plain",
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatal(err)
+		}
+
+		// copy the file into the tar stream
+		n, err := io.Copy(tw, bytes.NewBufferString(fileContents[i]))
+		if err != nil {
+			t.Fatal(err)
+		} else if n != size {
+			t.Fatal("size mismatch")
+		}
+	}
+
+	//post tar stream
+	url := srv.URL + "/bzz:/"
+	if encrypted {
+		url = url + "encrypt"
+	}
+	req, err := http.NewRequest("POST", url, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("Content-Type", "application/x-tar")
+	client := &http.Client{}
+	resp2, err := client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp2.StatusCode != http.StatusOK {
+		t.Fatalf("err %s", resp2.Status)
+	}
+	swarmHash, err := ioutil.ReadAll(resp2.Body)
+	resp2.Body.Close()
+	t.Logf("uploaded tarball successfully and got manifest address at %s", string(swarmHash))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now do a GET to get a tarball back
+	req, err = http.NewRequest("GET", fmt.Sprintf(srv.URL+"/bzz:/%s", string(swarmHash)), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("Accept", "application/x-tar")
+	resp2, err = client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp2.Body.Close()
+
+	file, err := ioutil.TempFile("", "swarm-downloaded-tarball")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(file.Name())
+	_, err = io.Copy(file, resp2.Body)
+	if err != nil {
+		t.Fatalf("error getting tarball: %v", err)
+	}
+	file.Sync()
+	file.Close()
+
+	tarFileHandle, err := os.Open(file.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(tarFileHandle)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			t.Fatalf("error reading tar stream: %s", err)
+		}
+		bb := make([]byte, hdr.Size)
+		_, err = tr.Read(bb)
+		if err != nil && err != io.EOF {
+			t.Fatal(err)
+		}
+		passed := false
+		for i, v := range fileNames {
+			if v == hdr.Name {
+				if string(bb) == fileContents[i] {
+					passed = true
+					break
+				}
+			}
+		}
+		if !passed {
+			t.Fatalf("file %s did not pass content assertion", hdr.Name)
+		}
+	}
+}
+
 // TestBzzRootRedirect tests that getting the root path of a manifest without
 // a trailing slash gets redirected to include the trailing slash so that
 // relative URLs work as expected.

swarm/api/manifest.go

@@ -18,6 +18,7 @@ package api
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -61,20 +62,20 @@ type ManifestList struct {
 }
 
 // NewManifest creates and stores a new, empty manifest
-func (a *API) NewManifest(toEncrypt bool) (storage.Address, error) {
+func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, error) {
 	var manifest Manifest
 	data, err := json.Marshal(&manifest)
 	if err != nil {
 		return nil, err
 	}
-	key, wait, err := a.Store(bytes.NewReader(data), int64(len(data)), toEncrypt)
+	key, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt)
-	wait()
+	wait(ctx)
 	return key, err
 }
 
 // Manifest hack for supporting Mutable Resource Updates from the bzz: scheme
 // see swarm/api/api.go:API.Get() for more information
-func (a *API) NewResourceManifest(resourceAddr string) (storage.Address, error) {
+func (a *API) NewResourceManifest(ctx context.Context, resourceAddr string) (storage.Address, error) {
 	var manifest Manifest
 	entry := ManifestEntry{
 		Hash:        resourceAddr,
@@ -85,7 +86,7 @@ func (a *API) NewResourceManifest(resourceAddr string) (storage.Address, error)
 	if err != nil {
 		return nil, err
 	}
-	key, _, err := a.Store(bytes.NewReader(data), int64(len(data)), false)
+	key, _, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false)
 	return key, err
 }
 
@@ -96,8 +97,8 @@ type ManifestWriter struct {
 	quitC chan bool
 }
 
-func (a *API) NewManifestWriter(addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
+func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
-	trie, err := loadManifest(a.fileStore, addr, quitC)
+	trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
 	if err != nil {
 		return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
 	}
@@ -105,9 +106,8 @@ func (a *API) NewManifestWriter(addr storage.Address, quitC chan bool) (*Manifes
 }
 
 // AddEntry stores the given data and adds the resulting key to the manifest
-func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Address, error) {
+func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (storage.Address, error) {
-
+	key, _, err := m.api.Store(ctx, data, e.Size, m.trie.encrypted)
-	key, _, err := m.api.Store(data, e.Size, m.trie.encrypted)
 	if err != nil {
 		return nil, err
 	}
@@ -136,8 +136,8 @@ type ManifestWalker struct {
 	quitC chan bool
 }
 
-func (a *API) NewManifestWalker(addr storage.Address, quitC chan bool) (*ManifestWalker, error) {
+func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWalker, error) {
-	trie, err := loadManifest(a.fileStore, addr, quitC)
+	trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
 	if err != nil {
 		return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
 	}
@@ -204,10 +204,10 @@ type manifestTrieEntry struct {
 	subtrie *manifestTrie
 }
 
-func loadManifest(fileStore *storage.FileStore, hash storage.Address, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 	log.Trace("manifest lookup", "key", hash)
 	// retrieve manifest via FileStore
-	manifestReader, isEncrypted := fileStore.Retrieve(hash)
+	manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash)
 	log.Trace("reader retrieved", "key", hash)
 	return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC)
 }
@@ -382,8 +382,12 @@ func (mt *manifestTrie) recalcAndStore() error {
 	}
 
 	sr := bytes.NewReader(manifest)
-	key, wait, err2 := mt.fileStore.Store(sr, int64(len(manifest)), mt.encrypted)
+	ctx := context.TODO()
-	wait()
+	key, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted)
+	if err2 != nil {
+		return err2
+	}
+	err2 = wait(ctx)
 	mt.ref = key
 	return err2
 }
@@ -391,7 +395,7 @@ func (mt *manifestTrie) recalcAndStore() error {
 func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
 	if entry.subtrie == nil {
 		hash := common.Hex2Bytes(entry.Hash)
-		entry.subtrie, err = loadManifest(mt.fileStore, hash, quitC)
+		entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC)
 		entry.Hash = "" // might not match, should be recalculated
 	}
 	return

swarm/api/storage.go

@@ -17,6 +17,7 @@
 package api
 
 import (
+	"context"
 	"path"
 
 	"github.com/ethereum/go-ethereum/swarm/storage"
@@ -45,8 +46,8 @@ func NewStorage(api *API) *Storage {
 // its content type
 //
 // DEPRECATED: Use the HTTP API instead
-func (s *Storage) Put(content, contentType string, toEncrypt bool) (storage.Address, func(), error) {
+func (s *Storage) Put(ctx context.Context, content string, contentType string, toEncrypt bool) (storage.Address, func(context.Context) error, error) {
-	return s.api.Put(content, contentType, toEncrypt)
+	return s.api.Put(ctx, content, contentType, toEncrypt)
 }
 
 // Get retrieves the content from bzzpath and reads the response in full
@@ -57,16 +58,16 @@ func (s *Storage) Put(content, contentType string, toEncrypt bool) (storage.Addr
 // size is resp.Size
 //
 // DEPRECATED: Use the HTTP API instead
-func (s *Storage) Get(bzzpath string) (*Response, error) {
+func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) {
 	uri, err := Parse(path.Join("bzz:/", bzzpath))
 	if err != nil {
 		return nil, err
 	}
-	addr, err := s.api.Resolve(uri)
+	addr, err := s.api.Resolve(ctx, uri)
 	if err != nil {
 		return nil, err
 	}
-	reader, mimeType, status, _, err := s.api.Get(addr, uri.Path)
+	reader, mimeType, status, _, err := s.api.Get(ctx, addr, uri.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -87,16 +88,16 @@ func (s *Storage) Get(bzzpath string) (*Response, error) {
 // and merge on  to it. creating an entry w conentType (mime)
 //
 // DEPRECATED: Use the HTTP API instead
-func (s *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
+func (s *Storage) Modify(ctx context.Context, rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
 	uri, err := Parse("bzz:/" + rootHash)
 	if err != nil {
 		return "", err
 	}
-	addr, err := s.api.Resolve(uri)
+	addr, err := s.api.Resolve(ctx, uri)
 	if err != nil {
 		return "", err
 	}
-	addr, err = s.api.Modify(addr, path, contentHash, contentType)
+	addr, err = s.api.Modify(ctx, addr, path, contentHash, contentType)
 	if err != nil {
 		return "", err
 	}

swarm/api/storage_test.go

@@ -17,6 +17,7 @@
 package api
 
 import (
+	"context"
 	"testing"
 )
 
@@ -31,18 +32,22 @@ func TestStoragePutGet(t *testing.T) {
 		content := "hello"
 		exp := expResponse(content, "text/plain", 0)
 		// exp := expResponse([]byte(content), "text/plain", 0)
-		bzzkey, wait, err := api.Put(content, exp.MimeType, toEncrypt)
+		ctx := context.TODO()
+		bzzkey, wait, err := api.Put(ctx, content, exp.MimeType, toEncrypt)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		err = wait(ctx)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		wait()
 		bzzhash := bzzkey.Hex()
 		// to check put against the API#Get
 		resp0 := testGet(t, api.api, bzzhash, "")
 		checkResponse(t, resp0, exp)
 
 		// check storage#Get
-		resp, err := api.Get(bzzhash)
+		resp, err := api.Get(context.TODO(), bzzhash)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}

swarm/bmt/bmt.go

@@ -117,10 +117,7 @@ func NewTreePool(hasher BaseHasherFunc, segmentCount, capacity int) *TreePool {
 	zerohashes[0] = zeros
 	h := hasher()
 	for i := 1; i < depth; i++ {
-		h.Reset()
+		zeros = doHash(h, nil, zeros, zeros)
-		h.Write(zeros)
-		h.Write(zeros)
-		zeros = h.Sum(nil)
 		zerohashes[i] = zeros
 	}
 	return &TreePool{
@@ -318,41 +315,19 @@ func (h *Hasher) Sum(b []byte) (r []byte) {
 // * if sequential write is used (can read sections)
 func (h *Hasher) sum(b []byte, release, section bool) (r []byte) {
 	t := h.bmt
-	h.finalise(section)
+	bh := h.pool.hasher()
-	if t.offset > 0 { // get the last node (double segment)
+	go h.writeSection(t.cur, t.section, true)
-
-		// padding the segment  with zero
-		copy(t.segment[t.offset:], h.pool.zerohashes[0])
-	}
-	if section {
-		if t.cur%2 == 1 {
-			// if just finished current segment, copy it to the right half of the chunk
-			copy(t.section[h.pool.SegmentSize:], t.segment)
-		} else {
-			// copy segment to front of section, zero pad the right half
-			copy(t.section, t.segment)
-			copy(t.section[h.pool.SegmentSize:], h.pool.zerohashes[0])
-		}
-		h.writeSection(t.cur, t.section)
-	} else {
-		// TODO: h.writeSegment(t.cur, t.segment)
-		panic("SegmentWriter not implemented")
-	}
 	bmtHash := <-t.result
 	span := t.span
-
+	// fmt.Println(t.draw(bmtHash))
 	if release {
 		h.releaseTree()
 	}
-	// sha3(span + BMT(pure_chunk))
+	// b + sha3(span + BMT(pure_chunk))
 	if span == nil {
-		return bmtHash
+		return append(b, bmtHash...)
 	}
-	bh := h.pool.hasher()
+	return doHash(bh, b, span, bmtHash)
-	bh.Reset()
-	bh.Write(span)
-	bh.Write(bmtHash)
-	return bh.Sum(b)
 }
 
 // Hasher implements the SwarmHash interface
@@ -367,37 +342,41 @@ func (h *Hasher) Write(b []byte) (int, error) {
 		return 0, nil
 	}
 	t := h.bmt
-	need := (h.pool.SegmentCount - t.cur) * h.pool.SegmentSize
+	secsize := 2 * h.pool.SegmentSize
-	if l < need {
+	// calculate length of missing bit to complete current open section
-		need = l
+	smax := secsize - t.offset
+	// if at the beginning of chunk or middle of the section
+	if t.offset < secsize {
+		// fill up current segment from buffer
+		copy(t.section[t.offset:], b)
+		// if input buffer consumed and open section not complete, then
+		// advance offset and return
+		if smax == 0 {
+			smax = secsize
 		}
-	// calculate missing bit to complete current open segment
+		if l <= smax {
-	rest := h.pool.SegmentSize - t.offset
+			t.offset += l
-	if need < rest {
+			return l, nil
-		rest = need
 		}
-	copy(t.segment[t.offset:], b[:rest])
-	need -= rest
-	size := (t.offset + rest) % h.pool.SegmentSize
-	// read full segments and the last possibly partial segment
-	for need > 0 {
-		// push all finished chunks we read
-		if t.cur%2 == 0 {
-			copy(t.section, t.segment)
 	} else {
-			copy(t.section[h.pool.SegmentSize:], t.segment)
+		if t.cur == h.pool.SegmentCount*2 {
-			h.writeSection(t.cur, t.section)
+			return 0, nil
 		}
-		size = h.pool.SegmentSize
-		if need < size {
-			size = need
 	}
-		copy(t.segment, b[rest:rest+size])
+	// read full segments and the last possibly partial segment from the input buffer
-		need -= size
+	for smax < l {
-		rest += size
+		// section complete; push to tree asynchronously
+		go h.writeSection(t.cur, t.section, false)
+		// reset section
+		t.section = make([]byte, secsize)
+		// copy from imput buffer at smax to right half of section
+		copy(t.section, b[smax:])
+		// advance cursor
 		t.cur++
+		// smax here represents successive offsets in the input buffer
+		smax += secsize
 	}
-	t.offset = size % h.pool.SegmentSize
+	t.offset = l - smax + secsize
 	return l, nil
 }
 
@@ -426,6 +405,8 @@ func (h *Hasher) releaseTree() {
 		t.span = nil
 		t.hash = nil
 		h.bmt = nil
+		t.section = make([]byte, h.pool.SegmentSize*2)
+		t.segment = make([]byte, h.pool.SegmentSize)
 		h.pool.release(t)
 	}
 }
@@ -435,29 +416,37 @@ func (h *Hasher) releaseTree() {
 // 	go h.run(h.bmt.leaves[i/2], h.pool.hasher(), i%2 == 0, s)
 // }
 
-// writeSection writes the hash of i/2-th segction into right level 1 node of the BMT tree
+// writeSection writes the hash of i-th section into level 1 node of the BMT tree
-func (h *Hasher) writeSection(i int, section []byte) {
+func (h *Hasher) writeSection(i int, section []byte, final bool) {
-	n := h.bmt.leaves[i/2]
+	// select the leaf node for the section
+	n := h.bmt.leaves[i]
 	isLeft := n.isLeft
 	n = n.parent
 	bh := h.pool.hasher()
-	bh.Write(section)
+	// hash the section
-	go func() {
+	s := doHash(bh, nil, section)
-		sum := bh.Sum(nil)
+	// write hash into parent node
-		if n == nil {
+	if final {
-			h.bmt.result <- sum
+		// for the last segment use writeFinalNode
-			return
+		h.writeFinalNode(1, n, bh, isLeft, s)
+	} else {
+		h.writeNode(n, bh, isLeft, s)
 	}
-		h.run(n, bh, isLeft, sum)
-	}()
 }
 
-// run pushes the data to the node
+// writeNode pushes the data to the node
 // if it is the first of 2 sisters written the routine returns
 // if it is the second, it calculates the hash and writes it
 // to the parent node recursively
-func (h *Hasher) run(n *node, bh hash.Hash, isLeft bool, s []byte) {
+func (h *Hasher) writeNode(n *node, bh hash.Hash, isLeft bool, s []byte) {
+	level := 1
 	for {
+		// at the root of the bmt just write the result to the result channel
+		if n == nil {
+			h.bmt.result <- s
+			return
+		}
+		// otherwise assign child hash to branc
 		if isLeft {
 			n.left = s
 		} else {
@@ -467,44 +456,68 @@ func (h *Hasher) run(n *node, bh hash.Hash, isLeft bool, s []byte) {
 		if n.toggle() {
 			return
 		}
-		// the second thread now can be sure both left and right children are written
+		// the thread coming later now can be sure both left and right children are written
-		// it calculates the hash of left|right and take it to the next level
+		// it calculates the hash of left|right and pushes it to the parent
-		bh.Reset()
+		s = doHash(bh, nil, n.left, n.right)
-		bh.Write(n.left)
-		bh.Write(n.right)
-		s = bh.Sum(nil)
-
-		// at the root of the bmt just write the result to the result channel
-		if n.parent == nil {
-			h.bmt.result <- s
-			return
-		}
-
-		// otherwise iterate on parent
 		isLeft = n.isLeft
 		n = n.parent
+		level++
 	}
 }
 
-// finalise is following the path starting from the final datasegment to the
+// writeFinalNode is following the path starting from the final datasegment to the
 // BMT root via parents
 // for unbalanced trees it fills in the missing right sister nodes using
 // the pool's lookup table for BMT subtree root hashes for all-zero sections
-func (h *Hasher) finalise(skip bool) {
+// otherwise behaves like `writeNode`
-	t := h.bmt
+func (h *Hasher) writeFinalNode(level int, n *node, bh hash.Hash, isLeft bool, s []byte) {
-	isLeft := t.cur%2 == 0
+
-	n := t.leaves[t.cur/2]
+	for {
-	for level := 0; n != nil; level++ {
+		// at the root of the bmt just write the result to the result channel
-		// when the final segment's path is going via left child node
+		if n == nil {
+			if s != nil {
+				h.bmt.result <- s
+			}
+			return
+		}
+		var noHash bool
+		if isLeft {
+			// coming from left sister branch
+			// when the final section's path is going via left child node
 			// we include an all-zero subtree hash for the right level and toggle the node.
 			// when the path is going through right child node, nothing to do
-		if isLeft && !skip {
 			n.right = h.pool.zerohashes[level]
-			n.toggle()
+			if s != nil {
+				n.left = s
+				// if a left final node carries a hash, it must be the first (and only thread)
+				// so the toggle is already in passive state no need no call
+				// yet thread needs to carry on pushing hash to parent
+			} else {
+				// if again first thread then propagate nil and calculate no hash
+				noHash = n.toggle()
+			}
+		} else {
+			// right sister branch
+			// if s is nil, then thread arrived first at previous node and here there will be two,
+			// so no need to do anything
+			if s != nil {
+				n.right = s
+				noHash = n.toggle()
+			} else {
+				noHash = true
+			}
+		}
+		// the child-thread first arriving will just continue resetting s to nil
+		// the second thread now can be sure both left and right children are written
+		// it calculates the hash of left|right and pushes it to the parent
+		if noHash {
+			s = nil
+		} else {
+			s = doHash(bh, nil, n.left, n.right)
 		}
-		skip = false
 		isLeft = n.isLeft
 		n = n.parent
+		level++
 	}
 }
 
@@ -525,6 +538,15 @@ func (n *node) toggle() bool {
 	return atomic.AddInt32(&n.state, 1)%2 == 1
 }
 
+// calculates the hash of the data using hash.Hash
+func doHash(h hash.Hash, b []byte, data ...[]byte) []byte {
+	h.Reset()
+	for _, v := range data {
+		h.Write(v)
+	}
+	return h.Sum(b)
+}
+
 func hashstr(b []byte) string {
 	end := len(b)
 	if end > 4 {

swarm/bmt/bmt_r.go

@@ -80,6 +80,5 @@ func (rh *RefHasher) hash(data []byte, length int) []byte {
 	}
 	rh.hasher.Reset()
 	rh.hasher.Write(section)
-	s := rh.hasher.Sum(nil)
+	return rh.hasher.Sum(nil)
-	return s
 }

swarm/bmt/bmt_test.go

@@ -34,12 +34,12 @@ import (
 // the actual data length generated (could be longer than max datalength of the BMT)
 const BufferSize = 4128
 
+var counts = []int{1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 32, 37, 42, 53, 63, 64, 65, 111, 127, 128}
+
+// calculates the Keccak256 SHA3 hash of the data
 func sha3hash(data ...[]byte) []byte {
 	h := sha3.NewKeccak256()
-	for _, v := range data {
+	return doHash(h, nil, data...)
-		h.Write(v)
-	}
-	return h.Sum(nil)
 }
 
 // TestRefHasher tests that the RefHasher computes the expected BMT hash for
@@ -129,32 +129,49 @@ func TestRefHasher(t *testing.T) {
 	}
 }
 
+// tests if hasher responds with correct hash
+func TestHasherEmptyData(t *testing.T) {
+	hasher := sha3.NewKeccak256
+	var data []byte
+	for _, count := range counts {
+		t.Run(fmt.Sprintf("%d_segments", count), func(t *testing.T) {
+			pool := NewTreePool(hasher, count, PoolSize)
+			defer pool.Drain(0)
+			bmt := New(pool)
+			rbmt := NewRefHasher(hasher, count)
+			refHash := rbmt.Hash(data)
+			expHash := Hash(bmt, nil, data)
+			if !bytes.Equal(expHash, refHash) {
+				t.Fatalf("hash mismatch with reference. expected %x, got %x", refHash, expHash)
+			}
+		})
+	}
+}
+
 func TestHasherCorrectness(t *testing.T) {
-	err := testHasher(testBaseHasher)
+	data := newData(BufferSize)
+	hasher := sha3.NewKeccak256
+	size := hasher().Size()
+
+	var err error
+	for _, count := range counts {
+		t.Run(fmt.Sprintf("segments_%v", count), func(t *testing.T) {
+			max := count * size
+			incr := 1
+			capacity := 1
+			pool := NewTreePool(hasher, count, capacity)
+			defer pool.Drain(0)
+			for n := 0; n <= max; n += incr {
+				incr = 1 + rand.Intn(5)
+				bmt := New(pool)
+				err = testHasherCorrectness(bmt, hasher, data, n, count)
 				if err != nil {
 					t.Fatal(err)
 				}
 			}
-
+		})
-func testHasher(f func(BaseHasherFunc, []byte, int, int) error) error {
-	data := newData(BufferSize)
-	hasher := sha3.NewKeccak256
-	size := hasher().Size()
-	counts := []int{1, 2, 3, 4, 5, 8, 16, 32, 64, 128}
-
-	var err error
-	for _, count := range counts {
-		max := count * size
-		incr := 1
-		for n := 1; n <= max; n += incr {
-			err = f(hasher, data, n, count)
-			if err != nil {
-				return err
 	}
 }
-	}
-	return nil
-}
 
 // Tests that the BMT hasher can be synchronously reused with poolsizes 1 and PoolSize
 func TestHasherReuse(t *testing.T) {
@@ -215,12 +232,69 @@ LOOP:
 	}
 }
 
-// helper function that creates  a tree pool
+// Tests BMT Hasher io.Writer interface is working correctly
-func testBaseHasher(hasher BaseHasherFunc, d []byte, n, count int) error {
+// even multiple short random write buffers
-	pool := NewTreePool(hasher, count, 1)
+func TestBMTHasherWriterBuffers(t *testing.T) {
+	hasher := sha3.NewKeccak256
+
+	for _, count := range counts {
+		t.Run(fmt.Sprintf("%d_segments", count), func(t *testing.T) {
+			errc := make(chan error)
+			pool := NewTreePool(hasher, count, PoolSize)
 			defer pool.Drain(0)
+			n := count * 32
 			bmt := New(pool)
-	return testHasherCorrectness(bmt, hasher, d, n, count)
+			data := newData(n)
+			rbmt := NewRefHasher(hasher, count)
+			refHash := rbmt.Hash(data)
+			expHash := Hash(bmt, nil, data)
+			if !bytes.Equal(expHash, refHash) {
+				t.Fatalf("hash mismatch with reference. expected %x, got %x", refHash, expHash)
+			}
+			attempts := 10
+			f := func() error {
+				bmt := New(pool)
+				bmt.Reset()
+				var buflen int
+				for offset := 0; offset < n; offset += buflen {
+					buflen = rand.Intn(n-offset) + 1
+					read, err := bmt.Write(data[offset : offset+buflen])
+					if err != nil {
+						return err
+					}
+					if read != buflen {
+						return fmt.Errorf("incorrect read. expected %v bytes, got %v", buflen, read)
+					}
+				}
+				hash := bmt.Sum(nil)
+				if !bytes.Equal(hash, expHash) {
+					return fmt.Errorf("hash mismatch. expected %x, got %x", hash, expHash)
+				}
+				return nil
+			}
+
+			for j := 0; j < attempts; j++ {
+				go func() {
+					errc <- f()
+				}()
+			}
+			timeout := time.NewTimer(2 * time.Second)
+			for {
+				select {
+				case err := <-errc:
+					if err != nil {
+						t.Fatal(err)
+					}
+					attempts--
+					if attempts == 0 {
+						return
+					}
+				case <-timeout.C:
+					t.Fatalf("timeout")
+				}
+			}
+		})
+	}
 }
 
 // helper function that compares reference and optimised implementations on

swarm/fuse/fuse_file.go

@@ -84,7 +84,7 @@ func (sf *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Gid = uint32(os.Getegid())
 
 	if sf.fileSize == -1 {
-		reader, _ := sf.mountInfo.swarmApi.Retrieve(sf.addr)
+		reader, _ := sf.mountInfo.swarmApi.Retrieve(ctx, sf.addr)
 		quitC := make(chan bool)
 		size, err := reader.Size(quitC)
 		if err != nil {
@@ -104,7 +104,7 @@ func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse
 	sf.lock.RLock()
 	defer sf.lock.RUnlock()
 	if sf.reader == nil {
-		sf.reader, _ = sf.mountInfo.swarmApi.Retrieve(sf.addr)
+		sf.reader, _ = sf.mountInfo.swarmApi.Retrieve(ctx, sf.addr)
 	}
 	buf := make([]byte, req.Size)
 	n, err := sf.reader.ReadAt(buf, req.Offset)

swarm/fuse/swarmfs_test.go

@@ -20,6 +20,7 @@ package fuse
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"flag"
 	"fmt"
@@ -110,7 +111,7 @@ func createTestFilesAndUploadToSwarm(t *testing.T, api *api.API, files map[strin
 	}
 
 	//upload directory to swarm and return hash
-	bzzhash, err := api.Upload(uploadDir, "", toEncrypt)
+	bzzhash, err := api.Upload(context.TODO(), uploadDir, "", toEncrypt)
 	if err != nil {
 		t.Fatalf("Error uploading directory %v: %vm encryption: %v", uploadDir, err, toEncrypt)
 	}

swarm/fuse/swarmfs_unix.go

@@ -19,6 +19,7 @@
 package fuse
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -104,7 +105,7 @@ func (swarmfs *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
 	}
 
 	log.Trace("swarmfs mount: getting manifest tree")
-	_, manifestEntryMap, err := swarmfs.swarmApi.BuildDirectoryTree(mhash, true)
+	_, manifestEntryMap, err := swarmfs.swarmApi.BuildDirectoryTree(context.TODO(), mhash, true)
 	if err != nil {
 		return nil, err
 	}

swarm/fuse/swarmfs_util.go

@@ -47,7 +47,7 @@ func externalUnmount(mountPoint string) error {
 }
 
 func addFileToSwarm(sf *SwarmFile, content []byte, size int) error {
-	fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(sf.mountInfo.LatestManifest, sf.path, sf.name, content, true)
+	fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(context.TODO(), sf.mountInfo.LatestManifest, sf.path, sf.name, content, true)
 	if err != nil {
 		return err
 	}
@@ -66,7 +66,7 @@ func addFileToSwarm(sf *SwarmFile, content []byte, size int) error {
 }
 
 func removeFileFromSwarm(sf *SwarmFile) error {
-	mkey, err := sf.mountInfo.swarmApi.RemoveFile(sf.mountInfo.LatestManifest, sf.path, sf.name, true)
+	mkey, err := sf.mountInfo.swarmApi.RemoveFile(context.TODO(), sf.mountInfo.LatestManifest, sf.path, sf.name, true)
 	if err != nil {
 		return err
 	}
@@ -102,7 +102,7 @@ func removeDirectoryFromSwarm(sd *SwarmDir) error {
 }
 
 func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error {
-	fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.addr, offset, length, true)
+	fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(context.TODO(), sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.addr, offset, length, true)
 	if err != nil {
 		return err
 	}

swarm/metrics/flags.go

@@ -81,6 +81,9 @@ func Setup(ctx *cli.Context) {
 			hosttag      = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name)
 		)
 
+		// Start system runtime metrics collection
+		go gethmetrics.CollectProcessMetrics(2 * time.Second)
+
 		if enableExport {
 			log.Info("Enabling swarm metrics export to InfluxDB")
 			go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{

swarm/network/networkid_test.go

@@ -0,0 +1,266 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package network
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+var (
+	currentNetworkID int
+	cnt              int
+	nodeMap          map[int][]discover.NodeID
+	kademlias        map[discover.NodeID]*Kademlia
+)
+
+const (
+	NumberOfNets = 4
+	MaxTimeout   = 6
+)
+
+func init() {
+	flag.Parse()
+	rand.Seed(time.Now().Unix())
+}
+
+/*
+Run the network ID test.
+The test creates one simulations.Network instance,
+a number of nodes, then connects nodes with each other in this network.
+
+Each node gets a network ID assigned according to the number of networks.
+Having more network IDs is just arbitrary in order to exclude
+false positives.
+
+Nodes should only connect with other nodes with the same network ID.
+After the setup phase, the test checks on each node if it has the
+expected node connections (excluding those not sharing the network ID).
+*/
+func TestNetworkID(t *testing.T) {
+	log.Debug("Start test")
+	//arbitrarily set the number of nodes. It could be any number
+	numNodes := 24
+	//the nodeMap maps all nodes (slice value) with the same network ID (key)
+	nodeMap = make(map[int][]discover.NodeID)
+	//set up the network and connect nodes
+	net, err := setupNetwork(numNodes)
+	if err != nil {
+		t.Fatalf("Error setting up network: %v", err)
+	}
+	defer func() {
+		//shutdown the snapshot network
+		log.Trace("Shutting down network")
+		net.Shutdown()
+	}()
+	//let's sleep to ensure all nodes are connected
+	time.Sleep(1 * time.Second)
+	//for each group sharing the same network ID...
+	for _, netIDGroup := range nodeMap {
+		log.Trace("netIDGroup size", "size", len(netIDGroup))
+		//...check that their size of the kademlia is of the expected size
+		//the assumption is that it should be the size of the group minus 1 (the node itself)
+		for _, node := range netIDGroup {
+			if kademlias[node].addrs.Size() != len(netIDGroup)-1 {
+				t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].addrs.Size(), len(netIDGroup)-1)
+			}
+			kademlias[node].EachAddr(nil, 0, func(addr OverlayAddr, _ int, _ bool) bool {
+				found := false
+				for _, nd := range netIDGroup {
+					p := ToOverlayAddr(nd.Bytes())
+					if bytes.Equal(p, addr.Address()) {
+						found = true
+					}
+				}
+				if !found {
+					t.Fatalf("Expected node not found for node %s", node.String())
+				}
+				return true
+			})
+		}
+	}
+	log.Info("Test terminated successfully")
+}
+
+// setup simulated network with bzz/discovery and pss services.
+// connects nodes in a circle
+// if allowRaw is set, omission of builtin pss encryption is enabled (see PssParams)
+func setupNetwork(numnodes int) (net *simulations.Network, err error) {
+	log.Debug("Setting up network")
+	quitC := make(chan struct{})
+	errc := make(chan error)
+	nodes := make([]*simulations.Node, numnodes)
+	if numnodes < 16 {
+		return nil, fmt.Errorf("Minimum sixteen nodes in network")
+	}
+	adapter := adapters.NewSimAdapter(newServices())
+	//create the network
+	net = simulations.NewNetwork(adapter, &simulations.NetworkConfig{
+		ID:             "NetworkIdTestNet",
+		DefaultService: "bzz",
+	})
+	log.Debug("Creating networks and nodes")
+
+	var connCount int
+
+	//create nodes and connect them to each other
+	for i := 0; i < numnodes; i++ {
+		log.Trace("iteration: ", "i", i)
+		nodeconf := adapters.RandomNodeConfig()
+		nodes[i], err = net.NewNodeWithConfig(nodeconf)
+		if err != nil {
+			return nil, fmt.Errorf("error creating node %d: %v", i, err)
+		}
+		err = net.Start(nodes[i].ID())
+		if err != nil {
+			return nil, fmt.Errorf("error starting node %d: %v", i, err)
+		}
+		client, err := nodes[i].Client()
+		if err != nil {
+			return nil, fmt.Errorf("create node %d rpc client fail: %v", i, err)
+		}
+		//now setup and start event watching in order to know when we can upload
+		ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout*time.Second)
+		defer watchCancel()
+		watchSubscriptionEvents(ctx, nodes[i].ID(), client, errc, quitC)
+		//on every iteration we connect to all previous ones
+		for k := i - 1; k >= 0; k-- {
+			connCount++
+			log.Debug(fmt.Sprintf("Connecting node %d with node %d; connection count is %d", i, k, connCount))
+			err = net.Connect(nodes[i].ID(), nodes[k].ID())
+			if err != nil {
+				if !strings.Contains(err.Error(), "already connected") {
+					return nil, fmt.Errorf("error connecting nodes: %v", err)
+				}
+			}
+		}
+	}
+	//now wait until the number of expected subscriptions has been finished
+	//`watchSubscriptionEvents` will write with a `nil` value to errc
+	for err := range errc {
+		if err != nil {
+			return nil, err
+		}
+		//`nil` received, decrement count
+		connCount--
+		log.Trace("count down", "cnt", connCount)
+		//all subscriptions received
+		if connCount == 0 {
+			close(quitC)
+			break
+		}
+	}
+	log.Debug("Network setup phase terminated")
+	return net, nil
+}
+
+func newServices() adapters.Services {
+	kademlias = make(map[discover.NodeID]*Kademlia)
+	kademlia := func(id discover.NodeID) *Kademlia {
+		if k, ok := kademlias[id]; ok {
+			return k
+		}
+		addr := NewAddrFromNodeID(id)
+		params := NewKadParams()
+		params.MinProxBinSize = 2
+		params.MaxBinSize = 3
+		params.MinBinSize = 1
+		params.MaxRetries = 1000
+		params.RetryExponent = 2
+		params.RetryInterval = 1000000
+		kademlias[id] = NewKademlia(addr.Over(), params)
+		return kademlias[id]
+	}
+	return adapters.Services{
+		"bzz": func(ctx *adapters.ServiceContext) (node.Service, error) {
+			addr := NewAddrFromNodeID(ctx.Config.ID)
+			hp := NewHiveParams()
+			hp.Discovery = false
+			cnt++
+			//assign the network ID
+			currentNetworkID = cnt % NumberOfNets
+			if ok := nodeMap[currentNetworkID]; ok == nil {
+				nodeMap[currentNetworkID] = make([]discover.NodeID, 0)
+			}
+			//add this node to the group sharing the same network ID
+			nodeMap[currentNetworkID] = append(nodeMap[currentNetworkID], ctx.Config.ID)
+			log.Debug("current network ID:", "id", currentNetworkID)
+			config := &BzzConfig{
+				OverlayAddr:  addr.Over(),
+				UnderlayAddr: addr.Under(),
+				HiveParams:   hp,
+				NetworkID:    uint64(currentNetworkID),
+			}
+			return NewBzz(config, kademlia(ctx.Config.ID), nil, nil, nil), nil
+		},
+	}
+}
+
+func watchSubscriptionEvents(ctx context.Context, id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) {
+	events := make(chan *p2p.PeerEvent)
+	sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
+	if err != nil {
+		log.Error(err.Error())
+		errc <- fmt.Errorf("error getting peer events for node %v: %s", id, err)
+		return
+	}
+	go func() {
+		defer func() {
+			sub.Unsubscribe()
+			log.Trace("watch subscription events: unsubscribe", "id", id)
+		}()
+
+		for {
+			select {
+			case <-quitC:
+				return
+			case <-ctx.Done():
+				select {
+				case errc <- ctx.Err():
+				case <-quitC:
+				}
+				return
+			case e := <-events:
+				if e.Type == p2p.PeerEventTypeAdd {
+					errc <- nil
+				}
+			case err := <-sub.Err():
+				if err != nil {
+					select {
+					case errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err):
+					case <-quitC:
+					}
+					return
+				}
+			}
+		}
+	}()
+}

swarm/network/stream/common_test.go

@@ -250,7 +250,7 @@ func (r *TestRegistry) APIs() []rpc.API {
 }
 
 func readAll(fileStore *storage.FileStore, hash []byte) (int64, error) {
-	r, _ := fileStore.Retrieve(hash)
+	r, _ := fileStore.Retrieve(context.TODO(), hash)
 	buf := make([]byte, 1024)
 	var n int
 	var total int64

swarm/network/stream/delivery_test.go

@@ -345,9 +345,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 	// here we distribute chunks of a random file into Stores of nodes 1 to nodes
 	rrFileStore := storage.NewFileStore(newRoundRobinStore(sim.Stores[1:]...), storage.NewFileStoreParams())
 	size := chunkCount * chunkSize
-	fileHash, wait, err := rrFileStore.Store(io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+	ctx := context.TODO()
+	fileHash, wait, err := rrFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
 	// wait until all chunks stored
-	wait()
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	err = wait(ctx)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -627,9 +631,13 @@ Loop:
 		hashes := make([]storage.Address, chunkCount)
 		for i := 0; i < chunkCount; i++ {
 			// create actual size real chunks
-			hash, wait, err := remoteFileStore.Store(io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false)
+			ctx := context.TODO()
+			hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false)
+			if err != nil {
+				b.Fatalf("expected no error. got %v", err)
+			}
 			// wait until all chunks stored
-			wait()
+			err = wait(ctx)
 			if err != nil {
 				b.Fatalf("expected no error. got %v", err)
 			}

swarm/network/stream/intervals_test.go

@@ -117,8 +117,12 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 
 	fileStore := storage.NewFileStore(sim.Stores[0], storage.NewFileStoreParams())
 	size := chunkCount * chunkSize
-	_, wait, err := fileStore.Store(io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+	ctx := context.TODO()
-	wait()
+	_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = wait(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}

swarm/network/stream/snapshot_retrieval_test.go

@@ -410,7 +410,7 @@ func runFileRetrievalTest(nodeCount int) error {
 		fileStore := registries[id].fileStore
 		//check all chunks
 		for i, hash := range conf.hashes {
-			reader, _ := fileStore.Retrieve(hash)
+			reader, _ := fileStore.Retrieve(context.TODO(), hash)
 			//check that we can read the file size and that it corresponds to the generated file size
 			if s, err := reader.Size(nil); err != nil || s != int64(len(randomFiles[i])) {
 				allSuccess = false
@@ -697,7 +697,7 @@ func runRetrievalTest(chunkCount int, nodeCount int) error {
 		fileStore := registries[id].fileStore
 		//check all chunks
 		for _, chnk := range conf.hashes {
-			reader, _ := fileStore.Retrieve(chnk)
+			reader, _ := fileStore.Retrieve(context.TODO(), chnk)
 			//assuming that reading the Size of the chunk is enough to know we found it
 			if s, err := reader.Size(nil); err != nil || s != chunkSize {
 				allSuccess = false
@@ -765,9 +765,13 @@ func uploadFilesToNodes(nodes []*simulations.Node) ([]storage.Address, []string,
 			return nil, nil, err
 		}
 		//store it (upload it) on the FileStore
-		rk, wait, err := fileStore.Store(strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false)
+		ctx := context.TODO()
+		rk, wait, err := fileStore.Store(ctx, strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false)
 		log.Debug("Uploaded random string file to node")
-		wait()
+		if err != nil {
+			return nil, nil, err
+		}
+		err = wait(ctx)
 		if err != nil {
 			return nil, nil, err
 		}

swarm/network/stream/snapshot_sync_test.go

@@ -581,8 +581,12 @@ func uploadFileToSingleNodeStore(id discover.NodeID, chunkCount int) ([]storage.
 	fileStore := storage.NewFileStore(lstore, storage.NewFileStoreParams())
 	var rootAddrs []storage.Address
 	for i := 0; i < chunkCount; i++ {
-		rk, wait, err := fileStore.Store(io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		ctx := context.TODO()
-		wait()
+		rk, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		if err != nil {
+			return nil, err
+		}
+		err = wait(ctx)
 		if err != nil {
 			return nil, err
 		}

swarm/network/stream/syncer_test.go

@@ -202,9 +202,12 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 		// here we distribute chunks of a random file into stores 1...nodes
 		rrFileStore := storage.NewFileStore(newRoundRobinStore(sim.Stores[1:]...), storage.NewFileStoreParams())
 		size := chunkCount * chunkSize
-		_, wait, err := rrFileStore.Store(io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		_, wait, err := rrFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		if err != nil {
+			t.Fatal(err.Error())
+		}
 		// need to wait cos we then immediately collect the relevant bin content
-		wait()
+		wait(ctx)
 		if err != nil {
 			t.Fatal(err.Error())
 		}

swarm/network_test.go

@@ -508,14 +508,15 @@ func uploadFile(swarm *Swarm) (storage.Address, string, error) {
 	// File data is very short, but it is ensured that its
 	// uniqueness is very certain.
 	data := fmt.Sprintf("test content %s %x", time.Now().Round(0), b)
-	k, wait, err := swarm.api.Put(data, "text/plain", false)
+	ctx := context.TODO()
+	k, wait, err := swarm.api.Put(ctx, data, "text/plain", false)
 	if err != nil {
 		return nil, "", err
 	}
 	if wait != nil {
-		wait()
+		err = wait(ctx)
 	}
-	return k, data, nil
+	return k, data, err
 }
 
 // retrieve is the function that is used for checking the availability of
@@ -570,7 +571,7 @@ func retrieve(
 
 				log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount))
 
-				r, _, _, _, err := swarm.api.Get(f.addr, "/")
+				r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/")
 				if err != nil {
 					errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err)
 					return

swarm/pss/handshake.go

@@ -385,7 +385,7 @@ func (ctl *HandshakeController) sendKey(pubkeyid string, topic *Topic, keycount
 	// generate new keys to send
 	for i := 0; i < len(recvkeyids); i++ {
 		var err error
-		recvkeyids[i], err = ctl.pss.generateSymmetricKey(*topic, to, true)
+		recvkeyids[i], err = ctl.pss.GenerateSymmetricKey(*topic, to, true)
 		if err != nil {
 			return []string{}, fmt.Errorf("set receive symkey fail (pubkey %x topic %x): %v", pubkeyid, topic, err)
 		}

swarm/pss/notify/notify.go

@@ -0,0 +1,394 @@
+package notify
+
+import (
+	"crypto/ecdsa"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/swarm/pss"
+)
+
+const (
+	// sent from requester to updater to request start of notifications
+	MsgCodeStart = iota
+
+	// sent from updater to requester, contains a notification plus a new symkey to replace the old
+	MsgCodeNotifyWithKey
+
+	// sent from updater to requester, contains a notification
+	MsgCodeNotify
+
+	// sent from requester to updater to request stop of notifications (currently unused)
+	MsgCodeStop
+	MsgCodeMax
+)
+
+const (
+	DefaultAddressLength = 1
+	symKeyLength         = 32 // this should be gotten from source
+)
+
+var (
+	// control topic is used before symmetric key issuance completes
+	controlTopic = pss.Topic{0x00, 0x00, 0x00, 0x01}
+)
+
+// when code is MsgCodeStart, Payload is address
+// when code is MsgCodeNotifyWithKey, Payload is notification | symkey
+// when code is MsgCodeNotify, Payload is notification
+// when code is MsgCodeStop, Payload is address
+type Msg struct {
+	Code       byte
+	Name       []byte
+	Payload    []byte
+	namestring string
+}
+
+// NewMsg creates a new notification message object
+func NewMsg(code byte, name string, payload []byte) *Msg {
+	return &Msg{
+		Code:       code,
+		Name:       []byte(name),
+		Payload:    payload,
+		namestring: name,
+	}
+}
+
+// NewMsgFromPayload decodes a serialized message payload into a new notification message object
+func NewMsgFromPayload(payload []byte) (*Msg, error) {
+	msg := &Msg{}
+	err := rlp.DecodeBytes(payload, msg)
+	if err != nil {
+		return nil, err
+	}
+	msg.namestring = string(msg.Name)
+	return msg, nil
+}
+
+// a notifier has one sendBin entry for each address space it sends messages to
+type sendBin struct {
+	address  pss.PssAddress
+	symKeyId string
+	count    int
+}
+
+// represents a single notification service
+// only subscription address bins that match the address of a notification client have entries.
+type notifier struct {
+	bins      map[string]*sendBin
+	topic     pss.Topic // identifies the resource for pss receiver
+	threshold int       // amount of address bytes used in bins
+	updateC   <-chan []byte
+	quitC     chan struct{}
+}
+
+func (n *notifier) removeSubscription() {
+	n.quitC <- struct{}{}
+}
+
+// represents an individual subscription made by a public key at a specific address/neighborhood
+type subscription struct {
+	pubkeyId string
+	address  pss.PssAddress
+	handler  func(string, []byte) error
+}
+
+// Controller is the interface to control, add and remove notification services and subscriptions
+type Controller struct {
+	pss           *pss.Pss
+	notifiers     map[string]*notifier
+	subscriptions map[string]*subscription
+	mu            sync.Mutex
+}
+
+// NewController creates a new Controller object
+func NewController(ps *pss.Pss) *Controller {
+	ctrl := &Controller{
+		pss:           ps,
+		notifiers:     make(map[string]*notifier),
+		subscriptions: make(map[string]*subscription),
+	}
+	ctrl.pss.Register(&controlTopic, ctrl.Handler)
+	return ctrl
+}
+
+// IsActive is used to check if a notification service exists for a specified id string
+// Returns true if exists, false if not
+func (c *Controller) IsActive(name string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.isActive(name)
+}
+
+func (c *Controller) isActive(name string) bool {
+	_, ok := c.notifiers[name]
+	return ok
+}
+
+// Subscribe is used by a client to request notifications from a notification service provider
+// It will create a MsgCodeStart message and send asymmetrically to the provider using its public key and routing address
+// The handler function is a callback that will be called when notifications are received
+// Fails if the request pss cannot be sent or if the update message could not be serialized
+func (c *Controller) Subscribe(name string, pubkey *ecdsa.PublicKey, address pss.PssAddress, handler func(string, []byte) error) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	msg := NewMsg(MsgCodeStart, name, c.pss.BaseAddr())
+	c.pss.SetPeerPublicKey(pubkey, controlTopic, &address)
+	pubkeyId := hexutil.Encode(crypto.FromECDSAPub(pubkey))
+	smsg, err := rlp.EncodeToBytes(msg)
+	if err != nil {
+		return err
+	}
+	err = c.pss.SendAsym(pubkeyId, controlTopic, smsg)
+	if err != nil {
+		return err
+	}
+	c.subscriptions[name] = &subscription{
+		pubkeyId: pubkeyId,
+		address:  address,
+		handler:  handler,
+	}
+	return nil
+}
+
+// Unsubscribe, perhaps unsurprisingly, undoes the effects of Subscribe
+// Fails if the subscription does not exist, if the request pss cannot be sent or if the update message could not be serialized
+func (c *Controller) Unsubscribe(name string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	sub, ok := c.subscriptions[name]
+	if !ok {
+		return fmt.Errorf("Unknown subscription '%s'", name)
+	}
+	msg := NewMsg(MsgCodeStop, name, sub.address)
+	smsg, err := rlp.EncodeToBytes(msg)
+	if err != nil {
+		return err
+	}
+	err = c.pss.SendAsym(sub.pubkeyId, controlTopic, smsg)
+	if err != nil {
+		return err
+	}
+	delete(c.subscriptions, name)
+	return nil
+}
+
+// NewNotifier is used by a notification service provider to create a new notification service
+// It takes a name as identifier for the resource, a threshold indicating the granularity of the subscription address bin
+// It then starts an event loop which listens to the supplied update channel and executes notifications on channel receives
+// Fails if a notifier already is registered on the name
+//func (c *Controller) NewNotifier(name string, threshold int, contentFunc func(string) ([]byte, error)) error {
+func (c *Controller) NewNotifier(name string, threshold int, updateC <-chan []byte) (func(), error) {
+	c.mu.Lock()
+	if c.isActive(name) {
+		c.mu.Unlock()
+		return nil, fmt.Errorf("Notification service %s already exists in controller", name)
+	}
+	quitC := make(chan struct{})
+	c.notifiers[name] = &notifier{
+		bins:      make(map[string]*sendBin),
+		topic:     pss.BytesToTopic([]byte(name)),
+		threshold: threshold,
+		updateC:   updateC,
+		quitC:     quitC,
+		//contentFunc: contentFunc,
+	}
+	c.mu.Unlock()
+	go func() {
+		for {
+			select {
+			case <-quitC:
+				return
+			case data := <-updateC:
+				c.notify(name, data)
+			}
+		}
+	}()
+
+	return c.notifiers[name].removeSubscription, nil
+}
+
+// RemoveNotifier is used to stop a notification service.
+// It cancels the event loop listening to the notification provider's update channel
+func (c *Controller) RemoveNotifier(name string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	currentNotifier, ok := c.notifiers[name]
+	if !ok {
+		return fmt.Errorf("Unknown notification service %s", name)
+	}
+	currentNotifier.removeSubscription()
+	delete(c.notifiers, name)
+	return nil
+}
+
+// Notify is called by a notification service provider to issue a new notification
+// It takes the name of the notification service and the data to be sent.
+// It fails if a notifier with this name does not exist or if data could not be serialized
+// Note that it does NOT fail on failure to send a message
+func (c *Controller) notify(name string, data []byte) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if !c.isActive(name) {
+		return fmt.Errorf("Notification service %s doesn't exist", name)
+	}
+	msg := NewMsg(MsgCodeNotify, name, data)
+	smsg, err := rlp.EncodeToBytes(msg)
+	if err != nil {
+		return err
+	}
+	for _, m := range c.notifiers[name].bins {
+		log.Debug("sending pss notify", "name", name, "addr", fmt.Sprintf("%x", m.address), "topic", fmt.Sprintf("%x", c.notifiers[name].topic), "data", data)
+		go func(m *sendBin) {
+			err = c.pss.SendSym(m.symKeyId, c.notifiers[name].topic, smsg)
+			if err != nil {
+				log.Warn("Failed to send notify to addr %x: %v", m.address, err)
+			}
+		}(m)
+	}
+	return nil
+}
+
+// check if we already have the bin
+// if we do, retrieve the symkey from it and increment the count
+// if we dont make a new symkey and a new bin entry
+func (c *Controller) addToBin(ntfr *notifier, address []byte) (symKeyId string, pssAddress pss.PssAddress, err error) {
+
+	// parse the address from the message and truncate if longer than our bins threshold
+	if len(address) > ntfr.threshold {
+		address = address[:ntfr.threshold]
+	}
+
+	pssAddress = pss.PssAddress(address)
+	hexAddress := fmt.Sprintf("%x", address)
+	currentBin, ok := ntfr.bins[hexAddress]
+	if ok {
+		currentBin.count++
+		symKeyId = currentBin.symKeyId
+	} else {
+		symKeyId, err = c.pss.GenerateSymmetricKey(ntfr.topic, &pssAddress, false)
+		if err != nil {
+			return "", nil, err
+		}
+		ntfr.bins[hexAddress] = &sendBin{
+			address:  address,
+			symKeyId: symKeyId,
+			count:    1,
+		}
+	}
+	return symKeyId, pssAddress, nil
+}
+
+func (c *Controller) handleStartMsg(msg *Msg, keyid string) (err error) {
+
+	keyidbytes, err := hexutil.Decode(keyid)
+	if err != nil {
+		return err
+	}
+	pubkey, err := crypto.UnmarshalPubkey(keyidbytes)
+	if err != nil {
+		return err
+	}
+
+	// if name is not registered for notifications we will not react
+	currentNotifier, ok := c.notifiers[msg.namestring]
+	if !ok {
+		return fmt.Errorf("Subscribe attempted on unknown resource '%s'", msg.namestring)
+	}
+
+	// add to or open new bin
+	symKeyId, pssAddress, err := c.addToBin(currentNotifier, msg.Payload)
+	if err != nil {
+		return err
+	}
+
+	// add to address book for send initial notify
+	symkey, err := c.pss.GetSymmetricKey(symKeyId)
+	if err != nil {
+		return err
+	}
+	err = c.pss.SetPeerPublicKey(pubkey, controlTopic, &pssAddress)
+	if err != nil {
+		return err
+	}
+
+	// TODO this is set to zero-length byte pending decision on protocol for initial message, whether it should include message or not, and how to trigger the initial message so that current state of MRU is sent upon subscription
+	notify := []byte{}
+	replyMsg := NewMsg(MsgCodeNotifyWithKey, msg.namestring, make([]byte, len(notify)+symKeyLength))
+	copy(replyMsg.Payload, notify)
+	copy(replyMsg.Payload[len(notify):], symkey)
+	sReplyMsg, err := rlp.EncodeToBytes(replyMsg)
+	if err != nil {
+		return err
+	}
+	return c.pss.SendAsym(keyid, controlTopic, sReplyMsg)
+}
+
+func (c *Controller) handleNotifyWithKeyMsg(msg *Msg) error {
+	symkey := msg.Payload[len(msg.Payload)-symKeyLength:]
+	topic := pss.BytesToTopic(msg.Name)
+
+	// \TODO keep track of and add actual address
+	updaterAddr := pss.PssAddress([]byte{})
+	c.pss.SetSymmetricKey(symkey, topic, &updaterAddr, true)
+	c.pss.Register(&topic, c.Handler)
+	return c.subscriptions[msg.namestring].handler(msg.namestring, msg.Payload[:len(msg.Payload)-symKeyLength])
+}
+
+func (c *Controller) handleStopMsg(msg *Msg) error {
+	// if name is not registered for notifications we will not react
+	currentNotifier, ok := c.notifiers[msg.namestring]
+	if !ok {
+		return fmt.Errorf("Unsubscribe attempted on unknown resource '%s'", msg.namestring)
+	}
+
+	// parse the address from the message and truncate if longer than our bins' address length threshold
+	address := msg.Payload
+	if len(msg.Payload) > currentNotifier.threshold {
+		address = address[:currentNotifier.threshold]
+	}
+
+	// remove the entry from the bin if it exists, and remove the bin if it's the last remaining one
+	hexAddress := fmt.Sprintf("%x", address)
+	currentBin, ok := currentNotifier.bins[hexAddress]
+	if !ok {
+		return fmt.Errorf("found no active bin for address %s", hexAddress)
+	}
+	currentBin.count--
+	if currentBin.count == 0 { // if no more clients in this bin, remove it
+		delete(currentNotifier.bins, hexAddress)
+	}
+	return nil
+}
+
+// Handler is the pss topic handler to be used to process notification service messages
+// It should be registered in the pss of both to any notification service provides and clients using the service
+func (c *Controller) Handler(smsg []byte, p *p2p.Peer, asymmetric bool, keyid string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	log.Debug("notify controller handler", "keyid", keyid)
+
+	// see if the message is valid
+	msg, err := NewMsgFromPayload(smsg)
+	if err != nil {
+		return err
+	}
+
+	switch msg.Code {
+	case MsgCodeStart:
+		return c.handleStartMsg(msg, keyid)
+	case MsgCodeNotifyWithKey:
+		return c.handleNotifyWithKeyMsg(msg)
+	case MsgCodeNotify:
+		return c.subscriptions[msg.namestring].handler(msg.namestring, msg.Payload)
+	case MsgCodeStop:
+		return c.handleStopMsg(msg)
+	}
+
+	return fmt.Errorf("Invalid message code: %d", msg.Code)
+}

swarm/pss/notify/notify_test.go

@@ -0,0 +1,252 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/pss"
+	"github.com/ethereum/go-ethereum/swarm/state"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+)
+
+var (
+	loglevel = flag.Int("l", 3, "loglevel")
+	psses    map[string]*pss.Pss
+	w        *whisper.Whisper
+	wapi     *whisper.PublicWhisperAPI
+)
+
+func init() {
+	flag.Parse()
+	hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
+	hf := log.LvlFilterHandler(log.Lvl(*loglevel), hs)
+	h := log.CallerFileHandler(hf)
+	log.Root().SetHandler(h)
+
+	w = whisper.New(&whisper.DefaultConfig)
+	wapi = whisper.NewPublicWhisperAPI(w)
+	psses = make(map[string]*pss.Pss)
+}
+
+// Creates a client node and notifier node
+// Client sends pss notifications requests
+// notifier sends initial notification with symmetric key, and
+// second notification symmetrically encrypted
+func TestStart(t *testing.T) {
+	adapter := adapters.NewSimAdapter(newServices(false))
+	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
+		ID:             "0",
+		DefaultService: "bzz",
+	})
+	leftNodeConf := adapters.RandomNodeConfig()
+	leftNodeConf.Services = []string{"bzz", "pss"}
+	leftNode, err := net.NewNodeWithConfig(leftNodeConf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = net.Start(leftNode.ID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rightNodeConf := adapters.RandomNodeConfig()
+	rightNodeConf.Services = []string{"bzz", "pss"}
+	rightNode, err := net.NewNodeWithConfig(rightNodeConf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = net.Start(rightNode.ID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = net.Connect(rightNode.ID(), leftNode.ID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	leftRpc, err := leftNode.Client()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rightRpc, err := rightNode.Client()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var leftAddr string
+	err = leftRpc.Call(&leftAddr, "pss_baseAddr")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var rightAddr string
+	err = rightRpc.Call(&rightAddr, "pss_baseAddr")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var leftPub string
+	err = leftRpc.Call(&leftPub, "pss_getPublicKey")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var rightPub string
+	err = rightRpc.Call(&rightPub, "pss_getPublicKey")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rsrcName := "foo.eth"
+	rsrcTopic := pss.BytesToTopic([]byte(rsrcName))
+
+	// wait for kademlia table to populate
+	time.Sleep(time.Second)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
+	defer cancel()
+	rmsgC := make(chan *pss.APIMsg)
+	rightSub, err := rightRpc.Subscribe(ctx, "pss", rmsgC, "receive", controlTopic)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rightSub.Unsubscribe()
+
+	updateC := make(chan []byte)
+	updateMsg := []byte{}
+	ctrlClient := NewController(psses[rightPub])
+	ctrlNotifier := NewController(psses[leftPub])
+	ctrlNotifier.NewNotifier("foo.eth", 2, updateC)
+
+	pubkeybytes, err := hexutil.Decode(leftPub)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pubkey, err := crypto.UnmarshalPubkey(pubkeybytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	addrbytes, err := hexutil.Decode(leftAddr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctrlClient.Subscribe(rsrcName, pubkey, addrbytes, func(s string, b []byte) error {
+		if s != "foo.eth" || !bytes.Equal(updateMsg, b) {
+			t.Fatalf("unexpected result in client handler: '%s':'%x'", s, b)
+		}
+		log.Info("client handler receive", "s", s, "b", b)
+		return nil
+	})
+
+	var inMsg *pss.APIMsg
+	select {
+	case inMsg = <-rmsgC:
+	case <-ctx.Done():
+		t.Fatal(ctx.Err())
+	}
+
+	dMsg, err := NewMsgFromPayload(inMsg.Msg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if dMsg.namestring != rsrcName {
+		t.Fatalf("expected name '%s', got '%s'", rsrcName, dMsg.namestring)
+	}
+	if !bytes.Equal(dMsg.Payload[:len(updateMsg)], updateMsg) {
+		t.Fatalf("expected payload first %d bytes '%x', got '%x'", len(updateMsg), updateMsg, dMsg.Payload[:len(updateMsg)])
+	}
+	if len(updateMsg)+symKeyLength != len(dMsg.Payload) {
+		t.Fatalf("expected payload length %d, have %d", len(updateMsg)+symKeyLength, len(dMsg.Payload))
+	}
+
+	rightSubUpdate, err := rightRpc.Subscribe(ctx, "pss", rmsgC, "receive", rsrcTopic)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rightSubUpdate.Unsubscribe()
+
+	updateMsg = []byte("plugh")
+	updateC <- updateMsg
+	select {
+	case inMsg = <-rmsgC:
+	case <-ctx.Done():
+		log.Error("timed out waiting for msg", "topic", fmt.Sprintf("%x", rsrcTopic))
+		t.Fatal(ctx.Err())
+	}
+	dMsg, err = NewMsgFromPayload(inMsg.Msg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if dMsg.namestring != rsrcName {
+		t.Fatalf("expected name %s, got %s", rsrcName, dMsg.namestring)
+	}
+	if !bytes.Equal(dMsg.Payload, updateMsg) {
+		t.Fatalf("expected payload '%x', got '%x'", updateMsg, dMsg.Payload)
+	}
+
+}
+
+func newServices(allowRaw bool) adapters.Services {
+	stateStore := state.NewInmemoryStore()
+	kademlias := make(map[discover.NodeID]*network.Kademlia)
+	kademlia := func(id discover.NodeID) *network.Kademlia {
+		if k, ok := kademlias[id]; ok {
+			return k
+		}
+		addr := network.NewAddrFromNodeID(id)
+		params := network.NewKadParams()
+		params.MinProxBinSize = 2
+		params.MaxBinSize = 3
+		params.MinBinSize = 1
+		params.MaxRetries = 1000
+		params.RetryExponent = 2
+		params.RetryInterval = 1000000
+		kademlias[id] = network.NewKademlia(addr.Over(), params)
+		return kademlias[id]
+	}
+	return adapters.Services{
+		"pss": func(ctx *adapters.ServiceContext) (node.Service, error) {
+			ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second)
+			defer cancel()
+			keys, err := wapi.NewKeyPair(ctxlocal)
+			privkey, err := w.GetPrivateKey(keys)
+			pssp := pss.NewPssParams().WithPrivateKey(privkey)
+			pssp.MsgTTL = time.Second * 30
+			pssp.AllowRaw = allowRaw
+			pskad := kademlia(ctx.Config.ID)
+			ps, err := pss.NewPss(pskad, pssp)
+			if err != nil {
+				return nil, err
+			}
+			//psses[common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey))] = ps
+			psses[hexutil.Encode(crypto.FromECDSAPub(&privkey.PublicKey))] = ps
+			return ps, nil
+		},
+		"bzz": func(ctx *adapters.ServiceContext) (node.Service, error) {
+			addr := network.NewAddrFromNodeID(ctx.Config.ID)
+			hp := network.NewHiveParams()
+			hp.Discovery = false
+			config := &network.BzzConfig{
+				OverlayAddr:  addr.Over(),
+				UnderlayAddr: addr.Under(),
+				HiveParams:   hp,
+			}
+			return network.NewBzz(config, kademlia(ctx.Config.ID), stateStore, nil, nil), nil
+		},
+	}
+}

swarm/pss/protocol.go

@@ -172,6 +172,8 @@ func (p *Protocol) Handle(msg []byte, peer *p2p.Peer, asymmetric bool, keyid str
 		rw, err := p.AddPeer(peer, *p.topic, asymmetric, keyid)
 		if err != nil {
 			return err
+		} else if rw == nil {
+			return fmt.Errorf("handle called on nil MsgReadWriter for new key " + keyid)
 		}
 		vrw = rw.(*PssReadWriter)
 	}
@@ -181,8 +183,14 @@ func (p *Protocol) Handle(msg []byte, peer *p2p.Peer, asymmetric bool, keyid str
 		return fmt.Errorf("could not decode pssmsg")
 	}
 	if asymmetric {
+		if p.pubKeyRWPool[keyid] == nil {
+			return fmt.Errorf("handle called on nil MsgReadWriter for key " + keyid)
+		}
 		vrw = p.pubKeyRWPool[keyid].(*PssReadWriter)
 	} else {
+		if p.symKeyRWPool[keyid] == nil {
+			return fmt.Errorf("handle called on nil MsgReadWriter for key " + keyid)
+		}
 		vrw = p.symKeyRWPool[keyid].(*PssReadWriter)
 	}
 	vrw.injectMsg(pmsg)

swarm/pss/pss.go

@@ -41,7 +41,7 @@ import (
 
 const (
 	defaultPaddingByteSize     = 16
-	defaultMsgTTL              = time.Second * 120
+	DefaultMsgTTL              = time.Second * 120
 	defaultDigestCacheTTL      = time.Second * 10
 	defaultSymKeyCacheCapacity = 512
 	digestLength               = 32 // byte length of digest used for pss cache (currently same as swarm chunk hash)
@@ -94,7 +94,7 @@ type PssParams struct {
 // Sane defaults for Pss
 func NewPssParams() *PssParams {
 	return &PssParams{
-		MsgTTL:              defaultMsgTTL,
+		MsgTTL:              DefaultMsgTTL,
 		CacheTTL:            defaultDigestCacheTTL,
 		SymKeyCacheCapacity: defaultSymKeyCacheCapacity,
 	}
@@ -354,11 +354,11 @@ func (p *Pss) handlePssMsg(msg interface{}) error {
 	}
 	if int64(pssmsg.Expire) < time.Now().Unix() {
 		metrics.GetOrRegisterCounter("pss.expire", nil).Inc(1)
-		log.Warn("pss filtered expired message", "from", fmt.Sprintf("%x", p.Overlay.BaseAddr()), "to", fmt.Sprintf("%x", common.ToHex(pssmsg.To)))
+		log.Warn("pss filtered expired message", "from", common.ToHex(p.Overlay.BaseAddr()), "to", common.ToHex(pssmsg.To))
 		return nil
 	}
 	if p.checkFwdCache(pssmsg) {
-		log.Trace(fmt.Sprintf("pss relay block-cache match (process): FROM %x TO %x", p.Overlay.BaseAddr(), common.ToHex(pssmsg.To)))
+		log.Trace("pss relay block-cache match (process)", "from", common.ToHex(p.Overlay.BaseAddr()), "to", (common.ToHex(pssmsg.To)))
 		return nil
 	}
 	p.addFwdCache(pssmsg)
@@ -480,7 +480,7 @@ func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address *Ps
 }
 
 // Automatically generate a new symkey for a topic and address hint
-func (p *Pss) generateSymmetricKey(topic Topic, address *PssAddress, addToCache bool) (string, error) {
+func (p *Pss) GenerateSymmetricKey(topic Topic, address *PssAddress, addToCache bool) (string, error) {
 	keyid, err := p.w.GenerateSymKey()
 	if err != nil {
 		return "", err

swarm/pss/pss_test.go

@@ -470,7 +470,7 @@ func TestKeys(t *testing.T) {
 	}
 
 	// make a symmetric key that we will send to peer for encrypting messages to us
-	inkeyid, err := ps.generateSymmetricKey(topicobj, &addr, true)
+	inkeyid, err := ps.GenerateSymmetricKey(topicobj, &addr, true)
 	if err != nil {
 		t.Fatalf("failed to set 'our' incoming symmetric key")
 	}
@@ -1296,7 +1296,7 @@ func benchmarkSymKeySend(b *testing.B) {
 	topic := BytesToTopic([]byte("foo"))
 	to := make(PssAddress, 32)
 	copy(to[:], network.RandomAddr().Over())
-	symkeyid, err := ps.generateSymmetricKey(topic, &to, true)
+	symkeyid, err := ps.GenerateSymmetricKey(topic, &to, true)
 	if err != nil {
 		b.Fatalf("could not generate symkey: %v", err)
 	}
@@ -1389,7 +1389,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) {
 	for i := 0; i < int(keycount); i++ {
 		to := make(PssAddress, 32)
 		copy(to[:], network.RandomAddr().Over())
-		keyid, err = ps.generateSymmetricKey(topic, &to, true)
+		keyid, err = ps.GenerateSymmetricKey(topic, &to, true)
 		if err != nil {
 			b.Fatalf("cant generate symkey #%d: %v", i, err)
 		}
@@ -1471,7 +1471,7 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) {
 	topic := BytesToTopic([]byte("foo"))
 	for i := 0; i < int(keycount); i++ {
 		copy(addr[i], network.RandomAddr().Over())
-		keyid, err = ps.generateSymmetricKey(topic, &addr[i], true)
+		keyid, err = ps.GenerateSymmetricKey(topic, &addr[i], true)
 		if err != nil {
 			b.Fatalf("cant generate symkey #%d: %v", i, err)
 		}

swarm/storage/chunker.go

@@ -16,6 +16,7 @@
 package storage
 
 import (
+	"context"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -126,7 +127,7 @@ type TreeChunker struct {
 	The chunks are not meant to be validated by the chunker when joining. This
 	is because it is left to the DPA to decide which sources are trusted.
 */
-func TreeJoin(addr Address, getter Getter, depth int) *LazyChunkReader {
+func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
 	jp := &JoinerParams{
 		ChunkerParams: ChunkerParams{
 			chunkSize: DefaultChunkSize,
@@ -137,14 +138,14 @@ func TreeJoin(addr Address, getter Getter, depth int) *LazyChunkReader {
 		depth:  depth,
 	}
 
-	return NewTreeJoiner(jp).Join()
+	return NewTreeJoiner(jp).Join(ctx)
 }
 
 /*
 	When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes.
 	New chunks to store are store using the putter which the caller provides.
 */
-func TreeSplit(data io.Reader, size int64, putter Putter) (k Address, wait func(), err error) {
+func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (k Address, wait func(context.Context) error, err error) {
 	tsp := &TreeSplitterParams{
 		SplitterParams: SplitterParams{
 			ChunkerParams: ChunkerParams{
@@ -156,7 +157,7 @@ func TreeSplit(data io.Reader, size int64, putter Putter) (k Address, wait func(
 		},
 		size: size,
 	}
-	return NewTreeSplitter(tsp).Split()
+	return NewTreeSplitter(tsp).Split(ctx)
 }
 
 func NewTreeJoiner(params *JoinerParams) *TreeChunker {
@@ -224,7 +225,7 @@ func (tc *TreeChunker) decrementWorkerCount() {
 	tc.workerCount -= 1
 }
 
-func (tc *TreeChunker) Split() (k Address, wait func(), err error) {
+func (tc *TreeChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error) {
 	if tc.chunkSize <= 0 {
 		panic("chunker must be initialised")
 	}
@@ -380,7 +381,7 @@ type LazyChunkReader struct {
 	getter    Getter
 }
 
-func (tc *TreeChunker) Join() *LazyChunkReader {
+func (tc *TreeChunker) Join(ctx context.Context) *LazyChunkReader {
 	return &LazyChunkReader{
 		key:       tc.addr,
 		chunkSize: tc.chunkSize,

swarm/storage/chunker_test.go

@@ -18,6 +18,7 @@ package storage
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
@@ -81,7 +82,7 @@ func testRandomBrokenData(n int, tester *chunkerTester) {
 	putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash)
 
 	expectedError := fmt.Errorf("Broken reader")
-	addr, _, err := TreeSplit(brokendata, int64(n), putGetter)
+	addr, _, err := TreeSplit(context.TODO(), brokendata, int64(n), putGetter)
 	if err == nil || err.Error() != expectedError.Error() {
 		tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err)
 	}
@@ -104,20 +105,24 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester)
 	putGetter := newTestHasherStore(NewMapChunkStore(), hash)
 
 	var addr Address
-	var wait func()
+	var wait func(context.Context) error
 	var err error
+	ctx := context.TODO()
 	if usePyramid {
-		addr, wait, err = PyramidSplit(data, putGetter, putGetter)
+		addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter)
 	} else {
-		addr, wait, err = TreeSplit(data, int64(n), putGetter)
+		addr, wait, err = TreeSplit(ctx, data, int64(n), putGetter)
 	}
 	if err != nil {
 		tester.t.Fatalf(err.Error())
 	}
 	tester.t.Logf(" Key = %v\n", addr)
-	wait()
+	err = wait(ctx)
+	if err != nil {
+		tester.t.Fatalf(err.Error())
+	}
 
-	reader := TreeJoin(addr, putGetter, 0)
+	reader := TreeJoin(context.TODO(), addr, putGetter, 0)
 	output := make([]byte, n)
 	r, err := reader.Read(output)
 	if r != n || err != io.EOF {
@@ -200,11 +205,15 @@ func TestDataAppend(t *testing.T) {
 		chunkStore := NewMapChunkStore()
 		putGetter := newTestHasherStore(chunkStore, SHA3Hash)
 
-		addr, wait, err := PyramidSplit(data, putGetter, putGetter)
+		ctx := context.TODO()
+		addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter)
+		if err != nil {
+			tester.t.Fatalf(err.Error())
+		}
+		err = wait(ctx)
 		if err != nil {
 			tester.t.Fatalf(err.Error())
 		}
-		wait()
 
 		//create a append data stream
 		appendInput, found := tester.inputs[uint64(m)]
@@ -217,13 +226,16 @@ func TestDataAppend(t *testing.T) {
 		}
 
 		putGetter = newTestHasherStore(chunkStore, SHA3Hash)
-		newAddr, wait, err := PyramidAppend(addr, appendData, putGetter, putGetter)
+		newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter)
+		if err != nil {
+			tester.t.Fatalf(err.Error())
+		}
+		err = wait(ctx)
 		if err != nil {
 			tester.t.Fatalf(err.Error())
 		}
-		wait()
 
-		reader := TreeJoin(newAddr, putGetter, 0)
+		reader := TreeJoin(ctx, newAddr, putGetter, 0)
 		newOutput := make([]byte, n+m)
 		r, err := reader.Read(newOutput)
 		if r != (n + m) {
@@ -282,12 +294,16 @@ func benchmarkSplitJoin(n int, t *testing.B) {
 		data := testDataReader(n)
 
 		putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash)
-		key, wait, err := PyramidSplit(data, putGetter, putGetter)
+		ctx := context.TODO()
+		key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
-		wait()
+		err = wait(ctx)
-		reader := TreeJoin(key, putGetter, 0)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		reader := TreeJoin(ctx, key, putGetter, 0)
 		benchReadAll(reader)
 	}
 }
@@ -298,7 +314,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) {
 		data := testDataReader(n)
 		putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash)
 
-		_, _, err := TreeSplit(data, int64(n), putGetter)
+		_, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
@@ -311,7 +327,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) {
 		data := testDataReader(n)
 		putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash)
 
-		_, _, err := TreeSplit(data, int64(n), putGetter)
+		_, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
@@ -324,7 +340,7 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) {
 		data := testDataReader(n)
 		putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash)
 
-		_, _, err := PyramidSplit(data, putGetter, putGetter)
+		_, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
@@ -338,7 +354,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) {
 		data := testDataReader(n)
 		putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash)
 
-		_, _, err := PyramidSplit(data, putGetter, putGetter)
+		_, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
@@ -354,18 +370,25 @@ func benchmarkSplitAppendPyramid(n, m int, t *testing.B) {
 		chunkStore := NewMapChunkStore()
 		putGetter := newTestHasherStore(chunkStore, SHA3Hash)
 
-		key, wait, err := PyramidSplit(data, putGetter, putGetter)
+		ctx := context.TODO()
+		key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		err = wait(ctx)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
-		wait()
 
 		putGetter = newTestHasherStore(chunkStore, SHA3Hash)
-		_, wait, err = PyramidAppend(key, data1, putGetter, putGetter)
+		_, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		err = wait(ctx)
 		if err != nil {
 			t.Fatalf(err.Error())
 		}
-		wait()
 	}
 }
 

swarm/storage/filestore.go

@@ -17,6 +17,7 @@
 package storage
 
 import (
+	"context"
 	"io"
 )
 
@@ -78,18 +79,18 @@ func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore {
 // Chunk retrieval blocks on netStore requests with a timeout so reader will
 // report error if retrieval of chunks within requested range time out.
 // It returns a reader with the chunk data and whether the content was encrypted
-func (f *FileStore) Retrieve(addr Address) (reader *LazyChunkReader, isEncrypted bool) {
+func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool) {
 	isEncrypted = len(addr) > f.hashFunc().Size()
 	getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted)
-	reader = TreeJoin(addr, getter, 0)
+	reader = TreeJoin(ctx, addr, getter, 0)
 	return
 }
 
 // Public API. Main entry point for document storage directly. Used by the
 // FS-aware API and httpaccess
-func (f *FileStore) Store(data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(), err error) {
+func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) {
 	putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt)
-	return PyramidSplit(data, putter, putter)
+	return PyramidSplit(ctx, data, putter, putter)
 }
 
 func (f *FileStore) HashSize() int {

swarm/storage/filestore_test.go

@@ -18,6 +18,7 @@ package storage
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"io/ioutil"
 	"os"
@@ -49,12 +50,16 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) {
 	defer os.RemoveAll("/tmp/bzz")
 
 	reader, slice := generateRandomData(testDataSize)
-	key, wait, err := fileStore.Store(reader, testDataSize, toEncrypt)
+	ctx := context.TODO()
+	key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt)
 	if err != nil {
 		t.Errorf("Store error: %v", err)
 	}
-	wait()
+	err = wait(ctx)
-	resultReader, isEncrypted := fileStore.Retrieve(key)
+	if err != nil {
+		t.Fatalf("Store waitt error: %v", err.Error())
+	}
+	resultReader, isEncrypted := fileStore.Retrieve(context.TODO(), key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
 	}
@@ -72,7 +77,7 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) {
 	ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666)
 	ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666)
 	localStore.memStore = NewMemStore(NewDefaultStoreParams(), db)
-	resultReader, isEncrypted = fileStore.Retrieve(key)
+	resultReader, isEncrypted = fileStore.Retrieve(context.TODO(), key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
 	}
@@ -110,12 +115,16 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) {
 	}
 	fileStore := NewFileStore(localStore, NewFileStoreParams())
 	reader, slice := generateRandomData(testDataSize)
-	key, wait, err := fileStore.Store(reader, testDataSize, toEncrypt)
+	ctx := context.TODO()
+	key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt)
 	if err != nil {
 		t.Errorf("Store error: %v", err)
 	}
-	wait()
+	err = wait(ctx)
-	resultReader, isEncrypted := fileStore.Retrieve(key)
+	if err != nil {
+		t.Errorf("Store error: %v", err)
+	}
+	resultReader, isEncrypted := fileStore.Retrieve(context.TODO(), key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
 	}
@@ -134,7 +143,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) {
 	memStore.setCapacity(0)
 	// check whether it is, indeed, empty
 	fileStore.ChunkStore = memStore
-	resultReader, isEncrypted = fileStore.Retrieve(key)
+	resultReader, isEncrypted = fileStore.Retrieve(context.TODO(), key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
 	}
@@ -144,7 +153,7 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) {
 	// check how it works with localStore
 	fileStore.ChunkStore = localStore
 	//	localStore.dbStore.setCapacity(0)
-	resultReader, isEncrypted = fileStore.Retrieve(key)
+	resultReader, isEncrypted = fileStore.Retrieve(context.TODO(), key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
 	}

swarm/storage/hasherstore.go

@@ -17,6 +17,7 @@
 package storage
 
 import (
+	"context"
 	"fmt"
 	"sync"
 
@@ -126,9 +127,10 @@ func (h *hasherStore) Close() {
 // Wait returns when
 //    1) the Close() function has been called and
 //    2) all the chunks which has been Put has been stored
-func (h *hasherStore) Wait() {
+func (h *hasherStore) Wait(ctx context.Context) error {
 	<-h.closed
 	h.wg.Wait()
+	return nil
 }
 
 func (h *hasherStore) createHash(chunkData ChunkData) Address {

swarm/storage/hasherstore_test.go

@@ -18,6 +18,7 @@ package storage
 
 import (
 	"bytes"
+	"context"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm/storage/encryption"
@@ -60,7 +61,10 @@ func TestHasherStore(t *testing.T) {
 		hasherStore.Close()
 
 		// Wait until chunks are really stored
-		hasherStore.Wait()
+		err = hasherStore.Wait(context.TODO())
+		if err != nil {
+			t.Fatalf("Expected no error got \"%v\"", err)
+		}
 
 		// Get the first chunk
 		retrievedChunkData1, err := hasherStore.Get(key1)

swarm/storage/ldbstore_test.go

@@ -59,12 +59,12 @@ func newTestDbStore(mock bool, trusted bool) (*testDbStore, func(), error) {
 	}
 
 	cleanup := func() {
-		if err != nil {
+		if db != nil {
 			db.Close()
 		}
 		err = os.RemoveAll(dir)
 		if err != nil {
-			panic("db cleanup failed")
+			panic(fmt.Sprintf("db cleanup failed: %v", err))
 		}
 	}
 

swarm/storage/pyramid.go

@@ -17,6 +17,7 @@
 package storage
 
 import (
+	"context"
 	"encoding/binary"
 	"errors"
 	"io"
@@ -99,12 +100,12 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
 	When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes.
 	New chunks to store are store using the putter which the caller provides.
 */
-func PyramidSplit(reader io.Reader, putter Putter, getter Getter) (Address, func(), error) {
+func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
-	return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split()
+	return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx)
 }
 
-func PyramidAppend(addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(), error) {
+func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
-	return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append()
+	return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx)
 }
 
 // Entry to create a tree node
@@ -203,7 +204,7 @@ func (pc *PyramidChunker) decrementWorkerCount() {
 	pc.workerCount -= 1
 }
 
-func (pc *PyramidChunker) Split() (k Address, wait func(), err error) {
+func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error) {
 	log.Debug("pyramid.chunker: Split()")
 
 	pc.wg.Add(1)
@@ -235,7 +236,7 @@ func (pc *PyramidChunker) Split() (k Address, wait func(), err error) {
 
 }
 
-func (pc *PyramidChunker) Append() (k Address, wait func(), err error) {
+func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(context.Context) error, err error) {
 	log.Debug("pyramid.chunker: Append()")
 	// Load the right most unfinished tree chunks in every level
 	pc.loadTree()

swarm/storage/types.go

@@ -18,6 +18,7 @@ package storage
 
 import (
 	"bytes"
+	"context"
 	"crypto"
 	"crypto/rand"
 	"encoding/binary"
@@ -303,7 +304,7 @@ type Putter interface {
 	// Close is to indicate that no more chunk data will be Put on this Putter
 	Close()
 	// Wait returns if all data has been store and the Close() was called.
-	Wait()
+	Wait(context.Context) error
 }
 
 // Getter is an interface to retrieve a chunk's data by its reference

swarm/swarm.go

@@ -388,10 +388,9 @@ func (self *Swarm) Start(srv *p2p.Server) error {
 	// start swarm http proxy server
 	if self.config.Port != "" {
 		addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port)
-		go httpapi.StartHTTPServer(self.api, &httpapi.ServerConfig{
+		server := httpapi.NewServer(self.api, self.config.Cors)
-			Addr:       addr,
+
-			CorsString: self.config.Cors,
+		go server.ListenAndServe(addr)
-		})
 	}
 
 	log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port))

swarm/swarm_test.go

@@ -17,10 +17,13 @@
 package swarm
 
 import (
+	"context"
+	"encoding/hex"
 	"io/ioutil"
 	"math/rand"
 	"os"
 	"path"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -42,6 +45,13 @@ func TestNewSwarm(t *testing.T) {
 	// a simple rpc endpoint for testing dialing
 	ipcEndpoint := path.Join(dir, "TestSwarm.ipc")
 
+	// windows namedpipes are not on filesystem but on NPFS
+	if runtime.GOOS == "windows" {
+		b := make([]byte, 8)
+		rand.Read(b)
+		ipcEndpoint = `\\.\pipe\TestSwarm-` + hex.EncodeToString(b)
+	}
+
 	_, server, err := rpc.StartIPCEndpoint(ipcEndpoint, nil)
 	if err != nil {
 		t.Error(err)
@@ -338,15 +348,19 @@ func testLocalStoreAndRetrieve(t *testing.T, swarm *Swarm, n int, randomData boo
 	}
 	dataPut := string(slice)
 
-	k, wait, err := swarm.api.Store(strings.NewReader(dataPut), int64(len(dataPut)), false)
+	ctx := context.TODO()
+	k, wait, err := swarm.api.Store(ctx, strings.NewReader(dataPut), int64(len(dataPut)), false)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if wait != nil {
-		wait()
+		err = wait(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}
 
-	r, _ := swarm.api.Retrieve(k)
+	r, _ := swarm.api.Retrieve(context.TODO(), k)
 
 	d, err := ioutil.ReadAll(r)
 	if err != nil {

vendor/github.com/mohae/deepcopy/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Joel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/mohae/deepcopy/README.md

@@ -0,0 +1,8 @@
+deepCopy
+========
+[![GoDoc](https://godoc.org/github.com/mohae/deepcopy?status.svg)](https://godoc.org/github.com/mohae/deepcopy)[![Build Status](https://travis-ci.org/mohae/deepcopy.png)](https://travis-ci.org/mohae/deepcopy)
+
+DeepCopy makes deep copies of things: unexported field values are not copied.
+
+## Usage
+    cpy := deepcopy.Copy(orig)

vendor/github.com/mohae/deepcopy/deepcopy.go

@@ -0,0 +1,125 @@
+// deepcopy makes deep copies of things. A standard copy will copy the
+// pointers: deep copy copies the values pointed to.  Unexported field
+// values are not copied.
+//
+// Copyright (c)2014-2016, Joel Scoble (github.com/mohae), all rights reserved.
+// License: MIT, for more details check the included LICENSE file.
+package deepcopy
+
+import (
+	"reflect"
+	"time"
+)
+
+// Interface for delegating copy process to type
+type Interface interface {
+	DeepCopy() interface{}
+}
+
+// Iface is an alias to Copy; this exists for backwards compatibility reasons.
+func Iface(iface interface{}) interface{} {
+	return Copy(iface)
+}
+
+// Copy creates a deep copy of whatever is passed to it and returns the copy
+// in an interface{}.  The returned value will need to be asserted to the
+// correct type.
+func Copy(src interface{}) interface{} {
+	if src == nil {
+		return nil
+	}
+
+	// Make the interface a reflect.Value
+	original := reflect.ValueOf(src)
+
+	// Make a copy of the same type as the original.
+	cpy := reflect.New(original.Type()).Elem()
+
+	// Recursively copy the original.
+	copyRecursive(original, cpy)
+
+	// Return the copy as an interface.
+	return cpy.Interface()
+}
+
+// copyRecursive does the actual copying of the interface. It currently has
+// limited support for what it can handle. Add as needed.
+func copyRecursive(original, cpy reflect.Value) {
+	// check for implement deepcopy.Interface
+	if original.CanInterface() {
+		if copier, ok := original.Interface().(Interface); ok {
+			cpy.Set(reflect.ValueOf(copier.DeepCopy()))
+			return
+		}
+	}
+
+	// handle according to original's Kind
+	switch original.Kind() {
+	case reflect.Ptr:
+		// Get the actual value being pointed to.
+		originalValue := original.Elem()
+
+		// if  it isn't valid, return.
+		if !originalValue.IsValid() {
+			return
+		}
+		cpy.Set(reflect.New(originalValue.Type()))
+		copyRecursive(originalValue, cpy.Elem())
+
+	case reflect.Interface:
+		// If this is a nil, don't do anything
+		if original.IsNil() {
+			return
+		}
+		// Get the value for the interface, not the pointer.
+		originalValue := original.Elem()
+
+		// Get the value by calling Elem().
+		copyValue := reflect.New(originalValue.Type()).Elem()
+		copyRecursive(originalValue, copyValue)
+		cpy.Set(copyValue)
+
+	case reflect.Struct:
+		t, ok := original.Interface().(time.Time)
+		if ok {
+			cpy.Set(reflect.ValueOf(t))
+			return
+		}
+		// Go through each field of the struct and copy it.
+		for i := 0; i < original.NumField(); i++ {
+			// The Type's StructField for a given field is checked to see if StructField.PkgPath
+			// is set to determine if the field is exported or not because CanSet() returns false
+			// for settable fields.  I'm not sure why.  -mohae
+			if original.Type().Field(i).PkgPath != "" {
+				continue
+			}
+			copyRecursive(original.Field(i), cpy.Field(i))
+		}
+
+	case reflect.Slice:
+		if original.IsNil() {
+			return
+		}
+		// Make a new slice and copy each element.
+		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
+		for i := 0; i < original.Len(); i++ {
+			copyRecursive(original.Index(i), cpy.Index(i))
+		}
+
+	case reflect.Map:
+		if original.IsNil() {
+			return
+		}
+		cpy.Set(reflect.MakeMap(original.Type()))
+		for _, key := range original.MapKeys() {
+			originalValue := original.MapIndex(key)
+			copyValue := reflect.New(originalValue.Type()).Elem()
+			copyRecursive(originalValue, copyValue)
+			copyKey := Copy(key.Interface())
+			cpy.SetMapIndex(reflect.ValueOf(copyKey), copyValue)
+		}
+
+	default:
+		cpy.Set(original)
+	}
+}

vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -640,6 +640,16 @@ func (db *DB) tableNeedCompaction() bool {
 	return v.needCompaction()
 }
 
+// resumeWrite returns an indicator whether we should resume write operation if enough level0 files are compacted.
+func (db *DB) resumeWrite() bool {
+	v := db.s.version()
+	defer v.release()
+	if v.tLen(0) < db.s.o.GetWriteL0PauseTrigger() {
+		return true
+	}
+	return false
+}
+
 func (db *DB) pauseCompaction(ch chan<- struct{}) {
 	select {
 	case ch <- struct{}{}:
@@ -653,6 +663,7 @@ type cCmd interface {
 }
 
 type cAuto struct {
+	// Note for table compaction, an empty ackC represents it's a compaction waiting command.
 	ackC chan<- error
 }
 
@@ -765,8 +776,10 @@ func (db *DB) mCompaction() {
 }
 
 func (db *DB) tCompaction() {
-	var x cCmd
+	var (
-	var ackQ []cCmd
+		x           cCmd
+		ackQ, waitQ []cCmd
+	)
 
 	defer func() {
 		if x := recover(); x != nil {
@@ -778,6 +791,10 @@ func (db *DB) tCompaction() {
 			ackQ[i].ack(ErrClosed)
 			ackQ[i] = nil
 		}
+		for i := range waitQ {
+			waitQ[i].ack(ErrClosed)
+			waitQ[i] = nil
+		}
 		if x != nil {
 			x.ack(ErrClosed)
 		}
@@ -795,12 +812,25 @@ func (db *DB) tCompaction() {
 				return
 			default:
 			}
+			// Resume write operation as soon as possible.
+			if len(waitQ) > 0 && db.resumeWrite() {
+				for i := range waitQ {
+					waitQ[i].ack(nil)
+					waitQ[i] = nil
+				}
+				waitQ = waitQ[:0]
+			}
 		} else {
 			for i := range ackQ {
 				ackQ[i].ack(nil)
 				ackQ[i] = nil
 			}
 			ackQ = ackQ[:0]
+			for i := range waitQ {
+				waitQ[i].ack(nil)
+				waitQ[i] = nil
+			}
+			waitQ = waitQ[:0]
 			select {
 			case x = <-db.tcompCmdC:
 			case ch := <-db.tcompPauseC:
@@ -813,7 +843,11 @@ func (db *DB) tCompaction() {
 		if x != nil {
 			switch cmd := x.(type) {
 			case cAuto:
+				if cmd.ackC != nil {
+					waitQ = append(waitQ, x)
+				} else {
 					ackQ = append(ackQ, x)
+				}
 			case cRange:
 				x.ack(db.tableRangeCompaction(cmd.level, cmd.min, cmd.max))
 			default:

vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go

@@ -9,10 +9,12 @@ package storage
 import (
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -42,6 +44,30 @@ func (lock *fileStorageLock) Unlock() {
 	}
 }
 
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func writeFileSynced(filename string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	n, err := f.Write(data)
+	if err == nil && n < len(data) {
+		err = io.ErrShortWrite
+	}
+	if err1 := f.Sync(); err == nil {
+		err = err1
+	}
+	if err1 := f.Close(); err == nil {
+		err = err1
+	}
+	return err
+}
+
 const logSizeThreshold = 1024 * 1024 // 1 MiB
 
 // fileStorage is a file-system backed storage.
@@ -60,7 +86,7 @@ type fileStorage struct {
 	day  int
 }
 
-// OpenFile returns a new filesytem-backed storage implementation with the given
+// OpenFile returns a new filesystem-backed storage implementation with the given
 // path. This also acquire a file lock, so any subsequent attempt to open the
 // same path will fail.
 //
@@ -189,7 +215,8 @@ func (fs *fileStorage) doLog(t time.Time, str string) {
 	// write
 	fs.buf = append(fs.buf, []byte(str)...)
 	fs.buf = append(fs.buf, '\n')
-	fs.logw.Write(fs.buf)
+	n, _ := fs.logw.Write(fs.buf)
+	fs.logSize += int64(n)
 }
 
 func (fs *fileStorage) Log(str string) {
@@ -210,7 +237,46 @@ func (fs *fileStorage) log(str string) {
 	}
 }
 
-func (fs *fileStorage) SetMeta(fd FileDesc) (err error) {
+func (fs *fileStorage) setMeta(fd FileDesc) error {
+	content := fsGenName(fd) + "\n"
+	// Check and backup old CURRENT file.
+	currentPath := filepath.Join(fs.path, "CURRENT")
+	if _, err := os.Stat(currentPath); err == nil {
+		b, err := ioutil.ReadFile(currentPath)
+		if err != nil {
+			fs.log(fmt.Sprintf("backup CURRENT: %v", err))
+			return err
+		}
+		if string(b) == content {
+			// Content not changed, do nothing.
+			return nil
+		}
+		if err := writeFileSynced(currentPath+".bak", b, 0644); err != nil {
+			fs.log(fmt.Sprintf("backup CURRENT: %v", err))
+			return err
+		}
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num)
+	if err := writeFileSynced(path, []byte(content), 0644); err != nil {
+		fs.log(fmt.Sprintf("create CURRENT.%d: %v", fd.Num, err))
+		return err
+	}
+	// Replace CURRENT file.
+	if err := rename(path, currentPath); err != nil {
+		fs.log(fmt.Sprintf("rename CURRENT.%d: %v", fd.Num, err))
+		return err
+	}
+	// Sync root directory.
+	if err := syncDir(fs.path); err != nil {
+		fs.log(fmt.Sprintf("syncDir: %v", err))
+		return err
+	}
+	return nil
+}
+
+func (fs *fileStorage) SetMeta(fd FileDesc) error {
 	if !FileDescOk(fd) {
 		return ErrInvalidFile
 	}
@@ -223,44 +289,10 @@ func (fs *fileStorage) SetMeta(fd FileDesc) (err error) {
 	if fs.open < 0 {
 		return ErrClosed
 	}
-	defer func() {
+	return fs.setMeta(fd)
-		if err != nil {
-			fs.log(fmt.Sprintf("CURRENT: %v", err))
-		}
-	}()
-	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num)
-	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
-	if err != nil {
-		return
-	}
-	_, err = fmt.Fprintln(w, fsGenName(fd))
-	if err != nil {
-		fs.log(fmt.Sprintf("write CURRENT.%d: %v", fd.Num, err))
-		return
-	}
-	if err = w.Sync(); err != nil {
-		fs.log(fmt.Sprintf("flush CURRENT.%d: %v", fd.Num, err))
-		return
-	}
-	if err = w.Close(); err != nil {
-		fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, err))
-		return
-	}
-	if err != nil {
-		return
-	}
-	if err = rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
-		fs.log(fmt.Sprintf("rename CURRENT.%d: %v", fd.Num, err))
-		return
-	}
-	// Sync root directory.
-	if err = syncDir(fs.path); err != nil {
-		fs.log(fmt.Sprintf("syncDir: %v", err))
-	}
-	return
 }
 
-func (fs *fileStorage) GetMeta() (fd FileDesc, err error) {
+func (fs *fileStorage) GetMeta() (FileDesc, error) {
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
 	if fs.open < 0 {
@@ -268,7 +300,7 @@ func (fs *fileStorage) GetMeta() (fd FileDesc, err error) {
 	}
 	dir, err := os.Open(fs.path)
 	if err != nil {
-		return
+		return FileDesc{}, err
 	}
 	names, err := dir.Readdirnames(0)
 	// Close the dir first before checking for Readdirnames error.
@@ -276,94 +308,134 @@ func (fs *fileStorage) GetMeta() (fd FileDesc, err error) {
 		fs.log(fmt.Sprintf("close dir: %v", ce))
 	}
 	if err != nil {
-		return
+		return FileDesc{}, err
 	}
-	// Find latest CURRENT file.
+	// Try this in order:
-	var rem []string
+	// - CURRENT.[0-9]+ ('pending rename' file, descending order)
-	var pend bool
+	// - CURRENT
-	var cerr error
+	// - CURRENT.bak
-	for _, name := range names {
+	//
-		if strings.HasPrefix(name, "CURRENT") {
+	// Skip corrupted file or file that point to a missing target file.
-			pend1 := len(name) > 7
+	type currentFile struct {
-			var pendNum int64
+		name string
-			// Make sure it is valid name for a CURRENT file, otherwise skip it.
+		fd   FileDesc
-			if pend1 {
-				if name[7] != '.' || len(name) < 9 {
-					fs.log(fmt.Sprintf("skipping %s: invalid file name", name))
-					continue
 	}
-				var e1 error
+	tryCurrent := func(name string) (*currentFile, error) {
-				if pendNum, e1 = strconv.ParseInt(name[8:], 10, 0); e1 != nil {
+		b, err := ioutil.ReadFile(filepath.Join(fs.path, name))
-					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", name, e1))
+		if err != nil {
-					continue
+			if os.IsNotExist(err) {
-				}
-			}
-			path := filepath.Join(fs.path, name)
-			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
-			if e1 != nil {
-				return FileDesc{}, e1
-			}
-			b, e1 := ioutil.ReadAll(r)
-			if e1 != nil {
-				r.Close()
-				return FileDesc{}, e1
-			}
-			var fd1 FileDesc
-			if len(b) < 1 || b[len(b)-1] != '\n' || !fsParseNamePtr(string(b[:len(b)-1]), &fd1) {
-				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", name))
-				if pend1 {
-					rem = append(rem, name)
-				}
-				if !pend1 || cerr == nil {
-					metaFd, _ := fsParseName(name)
-					cerr = &ErrCorrupted{
-						Fd:  metaFd,
-						Err: errors.New("leveldb/storage: corrupted or incomplete meta file"),
-					}
-				}
-			} else if pend1 && pendNum != fd1.Num {
-				fs.log(fmt.Sprintf("skipping %s: inconsistent pending-file num: %d vs %d", name, pendNum, fd1.Num))
-				rem = append(rem, name)
-			} else if fd1.Num < fd.Num {
-				fs.log(fmt.Sprintf("skipping %s: obsolete", name))
-				if pend1 {
-					rem = append(rem, name)
-				}
-			} else {
-				fd = fd1
-				pend = pend1
-			}
-			if err := r.Close(); err != nil {
-				fs.log(fmt.Sprintf("close %s: %v", name, err))
-			}
-		}
-	}
-	// Don't remove any files if there is no valid CURRENT file.
-	if fd.Zero() {
-		if cerr != nil {
-			err = cerr
-		} else {
 				err = os.ErrNotExist
 			}
-		return
+			return nil, err
 		}
-	if !fs.readOnly {
+		var fd FileDesc
-		// Rename pending CURRENT file to an effective CURRENT.
+		if len(b) < 1 || b[len(b)-1] != '\n' || !fsParseNamePtr(string(b[:len(b)-1]), &fd) {
-		if pend {
+			fs.log(fmt.Sprintf("%s: corrupted content: %q", name, b))
-			path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num)
+			err := &ErrCorrupted{
-			if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
+				Err: errors.New("leveldb/storage: corrupted or incomplete CURRENT file"),
-				fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", fd.Num, err))
+			}
+			return nil, err
+		}
+		if _, err := os.Stat(filepath.Join(fs.path, fsGenName(fd))); err != nil {
+			if os.IsNotExist(err) {
+				fs.log(fmt.Sprintf("%s: missing target file: %s", name, fd))
+				err = os.ErrNotExist
+			}
+			return nil, err
+		}
+		return &currentFile{name: name, fd: fd}, nil
+	}
+	tryCurrents := func(names []string) (*currentFile, error) {
+		var (
+			cur *currentFile
+			// Last corruption error.
+			lastCerr error
+		)
+		for _, name := range names {
+			var err error
+			cur, err = tryCurrent(name)
+			if err == nil {
+				break
+			} else if err == os.ErrNotExist {
+				// Fallback to the next file.
+			} else if isCorrupted(err) {
+				lastCerr = err
+				// Fallback to the next file.
+			} else {
+				// In case the error is due to permission, etc.
+				return nil, err
 			}
 		}
-		// Remove obsolete or incomplete pending CURRENT files.
+		if cur == nil {
-		for _, name := range rem {
+			err := os.ErrNotExist
-			path := filepath.Join(fs.path, name)
+			if lastCerr != nil {
-			if err := os.Remove(path); err != nil {
+				err = lastCerr
+			}
+			return nil, err
+		}
+		return cur, nil
+	}
+
+	// Try 'pending rename' files.
+	var nums []int64
+	for _, name := range names {
+		if strings.HasPrefix(name, "CURRENT.") && name != "CURRENT.bak" {
+			i, err := strconv.ParseInt(name[8:], 10, 64)
+			if err == nil {
+				nums = append(nums, i)
+			}
+		}
+	}
+	var (
+		pendCur   *currentFile
+		pendErr   = os.ErrNotExist
+		pendNames []string
+	)
+	if len(nums) > 0 {
+		sort.Sort(sort.Reverse(int64Slice(nums)))
+		pendNames = make([]string, len(nums))
+		for i, num := range nums {
+			pendNames[i] = fmt.Sprintf("CURRENT.%d", num)
+		}
+		pendCur, pendErr = tryCurrents(pendNames)
+		if pendErr != nil && pendErr != os.ErrNotExist && !isCorrupted(pendErr) {
+			return FileDesc{}, pendErr
+		}
+	}
+
+	// Try CURRENT and CURRENT.bak.
+	curCur, curErr := tryCurrents([]string{"CURRENT", "CURRENT.bak"})
+	if curErr != nil && curErr != os.ErrNotExist && !isCorrupted(curErr) {
+		return FileDesc{}, curErr
+	}
+
+	// pendCur takes precedence, but guards against obsolete pendCur.
+	if pendCur != nil && (curCur == nil || pendCur.fd.Num > curCur.fd.Num) {
+		curCur = pendCur
+	}
+
+	if curCur != nil {
+		// Restore CURRENT file to proper state.
+		if !fs.readOnly && (curCur.name != "CURRENT" || len(pendNames) != 0) {
+			// Ignore setMeta errors, however don't delete obsolete files if we
+			// catch error.
+			if err := fs.setMeta(curCur.fd); err == nil {
+				// Remove 'pending rename' files.
+				for _, name := range pendNames {
+					if err := os.Remove(filepath.Join(fs.path, name)); err != nil {
 						fs.log(fmt.Sprintf("remove %s: %v", name, err))
 					}
 				}
 			}
-	return
+		}
+		return curCur.fd, nil
+	}
+
+	// Nothing found.
+	if isCorrupted(pendErr) {
+		return FileDesc{}, pendErr
+	}
+	return FileDesc{}, curErr
 }
 
 func (fs *fileStorage) List(ft FileType) (fds []FileDesc, err error) {

vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go

@@ -67,13 +67,25 @@ func isErrInvalid(err error) bool {
 	if err == os.ErrInvalid {
 		return true
 	}
+	// Go < 1.8
 	if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL {
 		return true
 	}
+	// Go >= 1.8 returns *os.PathError instead
+	if patherr, ok := err.(*os.PathError); ok && patherr.Err == syscall.EINVAL {
+		return true
+	}
 	return false
 }
 
 func syncDir(name string) error {
+	// As per fsync manpage, Linux seems to expect fsync on directory, however
+	// some system don't support this, so we will ignore syscall.EINVAL.
+	//
+	// From fsync(2):
+	//   Calling fsync() does not necessarily ensure that the entry in the
+	//   directory containing the file has also reached disk. For that an
+	//   explicit fsync() on a file descriptor for the directory is also needed.
 	f, err := os.Open(name)
 	if err != nil {
 		return err

vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go

@@ -55,6 +55,14 @@ type ErrCorrupted struct {
 	Err error
 }
 
+func isCorrupted(err error) bool {
+	switch err.(type) {
+	case *ErrCorrupted:
+		return true
+	}
+	return false
+}
+
 func (e *ErrCorrupted) Error() string {
 	if !e.Fd.Zero() {
 		return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd)

vendor/vendor.json

@@ -291,6 +291,12 @@
 			"revision": "ad45545899c7b13c020ea92b2072220eefad42b8",
 			"revisionTime": "2015-03-14T17:03:34Z"
 		},
+		{
+			"checksumSHA1": "2jsbDTvwxafPp7FJjJ8IIFlTLjs=",
+			"path": "github.com/mohae/deepcopy",
+			"revision": "c48cc78d482608239f6c4c92a4abd87eb8761c90",
+			"revisionTime": "2017-09-29T03:49:55Z"
+		},
 		{
 			"checksumSHA1": "FYM/8R2CqS6PSNAoKl6X5gNJ20A=",
 			"path": "github.com/naoina/toml",
@@ -418,76 +424,76 @@
 			"revisionTime": "2017-07-05T02:17:15Z"
 		},
 		{
-			"checksumSHA1": "TJV50D0q8E3vtc90ibC+qOYdjrw=",
+			"checksumSHA1": "k6zbR5hiI10hkWtiK91rIY5s5/E=",
 			"path": "github.com/syndtr/goleveldb/leveldb",
-			"revision": "59047f74db0d042c8d8dd8e30bb030bc774a7d7a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-21T04:45:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=",
 			"path": "github.com/syndtr/goleveldb/leveldb/cache",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=",
 			"path": "github.com/syndtr/goleveldb/leveldb/comparer",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=",
 			"path": "github.com/syndtr/goleveldb/leveldb/errors",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=",
 			"path": "github.com/syndtr/goleveldb/leveldb/filter",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "weSsccMav4BCerDpSLzh3mMxAYo=",
 			"path": "github.com/syndtr/goleveldb/leveldb/iterator",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=",
 			"path": "github.com/syndtr/goleveldb/leveldb/journal",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "MtYY1b2234y/MlS+djL8tXVAcQs=",
 			"path": "github.com/syndtr/goleveldb/leveldb/memdb",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=",
 			"path": "github.com/syndtr/goleveldb/leveldb/opt",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
-			"checksumSHA1": "7H3fa12T7WoMAeXq1+qG5O7LD0w=",
+			"checksumSHA1": "ZnyuciM+R19NG8L5YS3TIJdo1e8=",
 			"path": "github.com/syndtr/goleveldb/leveldb/storage",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=",
 			"path": "github.com/syndtr/goleveldb/leveldb/table",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "V/Dh7NV0/fy/5jX1KaAjmGcNbzI=",
 			"path": "github.com/syndtr/goleveldb/leveldb/util",
-			"revision": "ae970a0732be3a1f5311da86118d37b9f4bd2a5a",
+			"revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445",
-			"revisionTime": "2018-05-02T07:23:49Z"
+			"revisionTime": "2018-07-08T03:05:51Z"
 		},
 		{
 			"checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=",