Merge branch 'swarm-network-rewrite' into swarm-network-rewrite-syncer

.gitignore

@@ -38,4 +38,4 @@ profile.cov
 /dashboard/assets/flow-typed
 /dashboard/assets/node_modules
 /dashboard/assets/stats.json
-/dashboard/assets/public/bundle.js
+/dashboard/assets/bundle.js

.travis.yml

@@ -40,6 +40,7 @@ matrix:
     - os: osx
       go: 1.9.x
       script:
+        - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - brew update
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse

cmd/geth/config.go

@@ -158,7 +158,7 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 	utils.RegisterEthService(stack, &cfg.Eth)
 
 	if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
-		utils.RegisterDashboardService(stack, &cfg.Dashboard)
+		utils.RegisterDashboardService(stack, &cfg.Dashboard, gitCommit)
 	}
 	// Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode
 	shhEnabled := enableWhisper(ctx)

cmd/geth/main.go

@@ -278,9 +278,12 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	// Start auxiliary services if enabled
 	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
 		// Mining only makes sense if a full Ethereum node is running
+		if ctx.GlobalBool(utils.LightModeFlag.Name) || ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
+			utils.Fatalf("Light clients do not support mining")
+		}
 		var ethereum *eth.Ethereum
 		if err := stack.Service(&ethereum); err != nil {
-			utils.Fatalf("ethereum service not running: %v", err)
+			utils.Fatalf("Ethereum service not running: %v", err)
 		}
 		// Use a reduced number of threads if requested
 		if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {

cmd/swarm/main.go

@@ -534,7 +534,8 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.
 			}
 		}
 
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled)
+		// In production, mockStore must be always nil.
+		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, nil)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {

cmd/utils/flags.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/fdlimit"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
@@ -721,10 +722,10 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
 // makeDatabaseHandles raises out the number of allowed file handles per process
 // for Geth and returns half of the allowance to assign to the database.
 func makeDatabaseHandles() int {
-	if err := raiseFdLimit(2048); err != nil {
+	if err := fdlimit.Raise(2048); err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
-	limit, err := getFdLimit()
+	limit, err := fdlimit.Current()
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
@@ -1103,9 +1104,9 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 }
 
 // RegisterDashboardService adds a dashboard to the stack.
-func RegisterDashboardService(stack *node.Node, cfg *dashboard.Config) {
+func RegisterDashboardService(stack *node.Node, cfg *dashboard.Config, commit string) {
 	stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		return dashboard.New(cfg)
+		return dashboard.New(cfg, commit)
 	})
 }
 

common/fdlimit/fdlimit_freebsd.go

@@ -16,7 +16,7 @@
 
 // +build freebsd
 
-package utils
+package fdlimit
 
 import "syscall"
 
@@ -24,9 +24,9 @@ import "syscall"
 // but Rlimit fields have type int64 on FreeBSD so it needs
 // an extra conversion.
 
-// raiseFdLimit tries to maximize the file descriptor allowance of this process
+// Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func raiseFdLimit(max uint64) error {
+func Raise(max uint64) error {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
@@ -43,9 +43,9 @@ func raiseFdLimit(max uint64) error {
 	return nil
 }
 
-// getFdLimit retrieves the number of file descriptors allowed to be opened by this
+// Current retrieves the number of file descriptors allowed to be opened by this
 // process.
-func getFdLimit() (int, error) {
+func Current() (int, error) {
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return 0, err
@@ -53,9 +53,9 @@ func getFdLimit() (int, error) {
 	return int(limit.Cur), nil
 }
 
-// getFdMaxLimit retrieves the maximum number of file descriptors this process is
+// Maximum retrieves the maximum number of file descriptors this process is
 // allowed to request for itself.
-func getFdMaxLimit() (int, error) {
+func Maximum() (int, error) {
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return 0, err

common/fdlimit/fdlimit_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-package utils
+package fdlimit
 
 import (
 	"fmt"
@@ -25,7 +25,7 @@ import (
 // per this process can be retrieved.
 func TestFileDescriptorLimits(t *testing.T) {
 	target := 4096
-	hardlimit, err := getFdMaxLimit()
+	hardlimit, err := Maximum()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -33,13 +33,13 @@ func TestFileDescriptorLimits(t *testing.T) {
 		t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target))
 	}
 
-	if limit, err := getFdLimit(); err != nil || limit <= 0 {
+	if limit, err := Current(); err != nil || limit <= 0 {
 		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
 	}
-	if err := raiseFdLimit(uint64(target)); err != nil {
+	if err := Raise(uint64(target)); err != nil {
 		t.Fatalf("failed to raise file allowance")
 	}
-	if limit, err := getFdLimit(); err != nil || limit < target {
+	if limit, err := Current(); err != nil || limit < target {
 		t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err)
 	}
 }

common/fdlimit/fdlimit_unix.go

@@ -16,13 +16,13 @@
 
 // +build linux darwin netbsd openbsd solaris
 
-package utils
+package fdlimit
 
 import "syscall"
 
-// raiseFdLimit tries to maximize the file descriptor allowance of this process
+// Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func raiseFdLimit(max uint64) error {
+func Raise(max uint64) error {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
@@ -39,9 +39,9 @@ func raiseFdLimit(max uint64) error {
 	return nil
 }
 
-// getFdLimit retrieves the number of file descriptors allowed to be opened by this
+// Current retrieves the number of file descriptors allowed to be opened by this
 // process.
-func getFdLimit() (int, error) {
+func Current() (int, error) {
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return 0, err
@@ -49,9 +49,9 @@ func getFdLimit() (int, error) {
 	return int(limit.Cur), nil
 }
 
-// getFdMaxLimit retrieves the maximum number of file descriptors this process is
+// Maximum retrieves the maximum number of file descriptors this process is
 // allowed to request for itself.
-func getFdMaxLimit() (int, error) {
+func Maximum() (int, error) {
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return 0, err

common/fdlimit/fdlimit_windows.go

@@ -14,13 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-package utils
+package fdlimit
 
 import "errors"
 
-// raiseFdLimit tries to maximize the file descriptor allowance of this process
+// Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func raiseFdLimit(max uint64) error {
+func Raise(max uint64) error {
 	// This method is NOP by design:
 	//  * Linux/Darwin counterparts need to manually increase per process limits
 	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
@@ -33,15 +33,15 @@ func raiseFdLimit(max uint64) error {
 	return nil
 }
 
-// getFdLimit retrieves the number of file descriptors allowed to be opened by this
+// Current retrieves the number of file descriptors allowed to be opened by this
 // process.
-func getFdLimit() (int, error) {
-	// Please see raiseFdLimit for the reason why we use hard coded 16K as the limit
+func Current() (int, error) {
+	// Please see Raise for the reason why we use hard coded 16K as the limit
 	return 16384, nil
 }
 
-// getFdMaxLimit retrieves the maximum number of file descriptors this process is
+// Maximum retrieves the maximum number of file descriptors this process is
 // allowed to request for itself.
-func getFdMaxLimit() (int, error) {
-	return getFdLimit()
+func Maximum() (int, error) {
+	return Current()
 }

dashboard/README.md

@@ -20,7 +20,7 @@ Normally the dashboard assets are bundled into Geth via `go-bindata` to avoid ex
 
 ```
 $ (cd dashboard/assets && ./node_modules/.bin/webpack --watch)
-$ geth --dashboard --dashboard.assets=dashboard/assets/public --vmodule=dashboard=5
+$ geth --dashboard --dashboard.assets=dashboard/assets --vmodule=dashboard=5
 ```
 
 To bundle up the final UI into Geth, run `go generate`:

dashboard/assets/components/Common.jsx

@@ -62,32 +62,4 @@ export type MenuProp = {|...ProvidedMenuProp, id: string|};
 // This way the mistyping is prevented.
 export const MENU: Map<string, {...MenuProp}> = new Map(menuSkeletons.map(({id, menu}) => ([id, {id, ...menu}])));
 
-type ProvidedSampleProp = {|limit: number|};
-const sampleSkeletons: Array<{|id: string, sample: ProvidedSampleProp|}> = [
-	{
-		id:     'memory',
-		sample: {
-			limit: 200,
-		},
-	}, {
-		id:     'traffic',
-		sample: {
-			limit: 200,
-		},
-	}, {
-		id:     'logs',
-		sample: {
-			limit: 200,
-		},
-	},
-];
-export type SampleProp = {|...ProvidedSampleProp, id: string|};
-export const SAMPLE: Map<string, {...SampleProp}> = new Map(sampleSkeletons.map(({id, sample}) => ([id, {id, ...sample}])));
-
 export const DURATION = 200;
-
-export const LENS: Map<string, string> = new Map([
-	'content',
-	...menuSkeletons.map(({id}) => id),
-	...sampleSkeletons.map(({id}) => id),
-].map(lens => [lens, lens]));

dashboard/assets/components/Dashboard.jsx

@@ -19,37 +19,99 @@
 import React, {Component} from 'react';
 
 import withStyles from 'material-ui/styles/withStyles';
-import {lensPath, view, set} from 'ramda';
 
 import Header from './Header';
 import Body from './Body';
-import {MENU, SAMPLE} from './Common';
-import type {Message, HomeMessage, LogsMessage, Chart} from '../types/message';
+import Footer from './Footer';
+import {MENU} from './Common';
 import type {Content} from '../types/content';
 
-// appender appends an array (A) to the end of another array (B) in the state.
-// lens is the path of B in the state, samples is A, and limit is the maximum size of the changed array.
+// deepUpdate updates an object corresponding to the given update data, which has
+// the shape of the same structure as the original object. updater also has the same
+// structure, except that it contains functions where the original data needs to be
+// updated. These functions are used to handle the update.
 //
-// appender retrieves a function, which overrides the state's value at lens, and returns with the overridden state.
-const appender = (lens, samples, limit) => (state) => {
-	const newSamples = [
-		...view(lens, state), // retrieves a specific value of the state at the given path (lens).
-		...samples,
-	];
-	// set is a function of ramda.js, which needs the path, the new value, the original state, and retrieves
-	// the altered state.
-	return set(
-		lens,
-		newSamples.slice(newSamples.length > limit ? newSamples.length - limit : 0),
-		state
-	);
+// Since the messages have the same shape as the state content, this approach allows
+// the generalization of the message handling. The only necessary thing is to set a
+// handler function for every path of the state in order to maximize the flexibility
+// of the update.
+const deepUpdate = (prev: Object, update: Object, updater: Object) => {
+	if (typeof update === 'undefined') {
+		// TODO (kurkomisi): originally this was deep copy, investigate it.
+		return prev;
+	}
+	if (typeof updater === 'function') {
+		return updater(prev, update);
+	}
+	const updated = {};
+	Object.keys(prev).forEach((key) => {
+		updated[key] = deepUpdate(prev[key], update[key], updater[key]);
+	});
+
+	return updated;
 };
-// Lenses for specific data fields in the state, used for a clearer deep update.
-// NOTE: This solution will be changed very likely.
-const memoryLens = lensPath(['content', 'home', 'memory']);
-const trafficLens = lensPath(['content', 'home', 'traffic']);
-const logLens = lensPath(['content', 'logs', 'log']);
-// styles retrieves the styles for the Dashboard component.
+
+// shouldUpdate returns the structure of a message. It is used to prevent unnecessary render
+// method triggerings. In the affected component's shouldComponentUpdate method it can be checked
+// whether the involved data was changed or not by checking the message structure.
+//
+// We could return the message itself too, but it's safer not to give access to it.
+const shouldUpdate = (msg: Object, updater: Object) => {
+	const su = {};
+	Object.keys(msg).forEach((key) => {
+		su[key] = typeof updater[key] !== 'function' ? shouldUpdate(msg[key], updater[key]) : true;
+	});
+
+	return su;
+};
+
+// appender is a state update generalization function, which appends the update data
+// to the existing data. limit defines the maximum allowed size of the created array.
+const appender = <T>(limit: number) => (prev: Array<T>, update: Array<T>) => [...prev, ...update].slice(-limit);
+
+// replacer is a state update generalization function, which replaces the original data.
+const replacer = <T>(prev: T, update: T) => update;
+
+// defaultContent is the initial value of the state content.
+const defaultContent: Content = {
+	general: {
+		version: null,
+		commit:  null,
+	},
+	home: {
+		memory:  [],
+		traffic: [],
+	},
+	chain:   {},
+	txpool:  {},
+	network: {},
+	system:  {},
+	logs:    {
+		log: [],
+	},
+};
+
+// updaters contains the state update generalization functions for each path of the state.
+// TODO (kurkomisi): Define a tricky type which embraces the content and the handlers.
+const updaters = {
+	general: {
+		version: replacer,
+		commit:  replacer,
+	},
+	home: {
+		memory:  appender(200),
+		traffic: appender(200),
+	},
+	chain:   null,
+	txpool:  null,
+	network: null,
+	system:  null,
+	logs:    {
+		log: appender(200),
+	},
+};
+
+// styles returns the styles for the Dashboard component.
 const styles = theme => ({
 	dashboard: {
 		display:    'flex',
@@ -61,15 +123,18 @@ const styles = theme => ({
 		overflow:   'hidden',
 	},
 });
+
 export type Props = {
 	classes: Object,
 };
+
 type State = {
 	active: string, // active menu
 	sideBar: boolean, // true if the sidebar is opened
-	content: $Shape<Content>, // the visualized data
-	shouldUpdate: Set<string> // labels for the components, which need to rerender based on the incoming message
+	content: Content, // the visualized data
+	shouldUpdate: Object // labels for the components, which need to rerender based on the incoming message
 };
+
 // Dashboard is the main component, which renders the whole page, makes connection with the server and
 // listens for messages. When there is an incoming message, updates the page's content correspondingly.
 class Dashboard extends Component<Props, State> {
@@ -78,8 +143,8 @@ class Dashboard extends Component<Props, State> {
 		this.state = {
 			active:       MENU.get('home').id,
 			sideBar:      true,
-			content:      {home: {memory: [], traffic: []}, logs: {log: []}},
-			shouldUpdate: new Set(),
+			content:      defaultContent,
+			shouldUpdate: {},
 		};
 	}
 
@@ -91,13 +156,14 @@ class Dashboard extends Component<Props, State> {
 	// reconnect establishes a websocket connection with the server, listens for incoming messages
 	// and tries to reconnect on connection loss.
 	reconnect = () => {
-		this.setState({
-			content: {home: {memory: [], traffic: []}, logs: {log: []}},
-		});
 		const server = new WebSocket(`${((window.location.protocol === 'https:') ? 'wss://' : 'ws://') + window.location.host}/api`);
+		server.onopen = () => {
+			this.setState({content: defaultContent, shouldUpdate: {}});
+		};
 		server.onmessage = (event) => {
-			const msg: Message = JSON.parse(event.data);
+			const msg: $Shape<Content> = JSON.parse(event.data);
 			if (!msg) {
+				console.error(`Incoming message is ${msg}`);
 				return;
 			}
 			this.update(msg);
@@ -107,56 +173,12 @@ class Dashboard extends Component<Props, State> {
 		};
 	};
 
-	// samples retrieves the raw data of a chart field from the incoming message.
-	samples = (chart: Chart) => {
-		let s = [];
-		if (chart.history) {
-			s = chart.history.map(({value}) => (value || 0)); // traffic comes without value at the beginning
-		}
-		if (chart.new) {
-			s = [...s, chart.new.value || 0];
-		}
-		return s;
-	};
-
-	// handleHome changes the home-menu related part of the state.
-	handleHome = (home: HomeMessage) => {
-		this.setState((prevState) => {
-			let newState = prevState;
-			newState.shouldUpdate = new Set();
-			if (home.memory) {
-				newState = appender(memoryLens, this.samples(home.memory), SAMPLE.get('memory').limit)(newState);
-				newState.shouldUpdate.add('memory');
-			}
-			if (home.traffic) {
-				newState = appender(trafficLens, this.samples(home.traffic), SAMPLE.get('traffic').limit)(newState);
-				newState.shouldUpdate.add('traffic');
-			}
-			return newState;
-		});
-	};
-
-	// handleLogs changes the logs-menu related part of the state.
-	handleLogs = (logs: LogsMessage) => {
-		this.setState((prevState) => {
-			let newState = prevState;
-			newState.shouldUpdate = new Set();
-			if (logs.log) {
-				newState = appender(logLens, [logs.log], SAMPLE.get('logs').limit)(newState);
-				newState.shouldUpdate.add('logs');
-			}
-			return newState;
-		});
-	};
-
-	// update analyzes the incoming message, and updates the charts' content correspondingly.
-	update = (msg: Message) => {
-		if (msg.home) {
-			this.handleHome(msg.home);
-		}
-		if (msg.logs) {
-			this.handleLogs(msg.logs);
-		}
+	// update updates the content corresponding to the incoming message.
+	update = (msg: $Shape<Content>) => {
+		this.setState(prevState => ({
+			content:      deepUpdate(prevState.content, msg, updaters),
+			shouldUpdate: shouldUpdate(msg, updaters),
+		}));
 	};
 
 	// changeContent sets the active label, which is used at the content rendering.
@@ -191,6 +213,13 @@ class Dashboard extends Component<Props, State> {
 					content={this.state.content}
 					shouldUpdate={this.state.shouldUpdate}
 				/>
+				<Footer
+					opened={this.state.sideBar}
+					openSideBar={this.openSideBar}
+					closeSideBar={this.closeSideBar}
+					general={this.state.content.general}
+					shouldUpdate={this.state.shouldUpdate}
+				/>
 			</div>
 		);
 	}

dashboard/assets/components/Footer.jsx

@@ -0,0 +1,80 @@
+// @flow
+
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+
+import withStyles from 'material-ui/styles/withStyles';
+import AppBar from 'material-ui/AppBar';
+import Toolbar from 'material-ui/Toolbar';
+import Typography from 'material-ui/Typography';
+
+import type {General} from '../types/content';
+
+// styles contains styles for the Header component.
+const styles = theme => ({
+	footer: {
+		backgroundColor: theme.palette.background.appBar,
+		color:           theme.palette.getContrastText(theme.palette.background.appBar),
+		zIndex:          theme.zIndex.appBar,
+	},
+	toolbar: {
+		paddingLeft:    theme.spacing.unit,
+		paddingRight:   theme.spacing.unit,
+		display:        'flex',
+		justifyContent: 'flex-end',
+	},
+	light: {
+		color: 'rgba(255, 255, 255, 0.54)',
+	},
+});
+export type Props = {
+	general: General,
+	classes: Object,
+};
+// TODO (kurkomisi): If the structure is appropriate, make an abstraction of the common parts with the Header.
+// Footer renders the header of the dashboard.
+class Footer extends Component<Props> {
+	shouldComponentUpdate(nextProps) {
+		return typeof nextProps.shouldUpdate.logs !== 'undefined';
+	}
+
+	info = (about: string, data: string) => (
+		<Typography type="caption" color="inherit">
+			<span className={this.props.classes.light}>{about}</span> {data}
+		</Typography>
+	);
+
+	render() {
+		const {classes, general} = this.props; // The classes property is injected by withStyles().
+		const geth = general.version ? this.info('Geth', general.version) : null;
+		const commit = general.commit ? this.info('Commit', general.commit.substring(0, 7)) : null;
+
+		return (
+			<AppBar position="static" className={classes.footer}>
+				<Toolbar className={classes.toolbar}>
+					<div>
+						{geth}
+						{commit}
+					</div>
+				</Toolbar>
+			</AppBar>
+		);
+	}
+}
+
+export default withStyles(styles)(Footer);

dashboard/assets/components/Home.jsx

@@ -22,13 +22,13 @@ import withTheme from 'material-ui/styles/withTheme';
 import {LineChart, AreaChart, Area, YAxis, CartesianGrid, Line} from 'recharts';
 
 import ChartGrid from './ChartGrid';
-import type {ChartEntry} from '../types/message';
+import type {ChartEntry} from '../types/content';
 
 export type Props = {
     theme: Object,
     memory: Array<ChartEntry>,
     traffic: Array<ChartEntry>,
-    shouldUpdate: Object,
+	shouldUpdate: Object,
 };
 // Home renders the home content.
 class Home extends Component<Props> {
@@ -40,11 +40,16 @@ class Home extends Component<Props> {
 	}
 
 	shouldComponentUpdate(nextProps) {
-		return nextProps.shouldUpdate.has('memory') || nextProps.shouldUpdate.has('traffic');
+		return typeof nextProps.shouldUpdate.home !== 'undefined';
 	}
 
+	memoryColor: Object;
+	trafficColor: Object;
+
 	render() {
-		const {memory, traffic} = this.props;
+		let {memory, traffic} = this.props;
+		memory = memory.map(({value}) => (value || 0));
+		traffic = traffic.map(({value}) => (value || 0));
 
 		return (
 			<ChartGrid spacing={24}>

dashboard/assets/package.json

@@ -1,7 +1,7 @@
 {
     "dependencies": {
         "babel-core": "^6.26.0",
-        "babel-eslint": "^8.0.3",
+        "babel-eslint": "^8.1.2",
         "babel-loader": "^7.1.2",
         "babel-plugin-transform-class-properties": "^6.24.1",
         "babel-plugin-transform-decorators-legacy": "^1.3.4",
@@ -12,28 +12,28 @@
         "babel-preset-stage-0": "^6.24.1",
         "babel-runtime": "^6.26.0",
         "classnames": "^2.2.5",
-        "css-loader": "^0.28.7",
-        "eslint": "^4.13.1",
+        "css-loader": "^0.28.8",
+        "eslint": "^4.15.0",
         "eslint-config-airbnb": "^16.1.0",
         "eslint-loader": "^1.9.0",
         "eslint-plugin-import": "^2.8.0",
         "eslint-plugin-jsx-a11y": "^6.0.3",
         "eslint-plugin-react": "^7.5.1",
-        "eslint-plugin-flowtype": "^2.40.1",
+        "eslint-plugin-flowtype": "^2.41.0",
         "file-loader": "^1.1.6",
-        "flow-bin": "^0.61.0",
+        "flow-bin": "^0.63.1",
         "flow-bin-loader": "^1.0.2",
         "flow-typed": "^2.2.3",
         "material-ui": "^1.0.0-beta.24",
         "material-ui-icons": "^1.0.0-beta.17",
         "path": "^0.12.7",
-        "ramda": "^0.25.0",
         "react": "^16.2.0",
         "react-dom": "^16.2.0",
         "react-fa": "^5.0.0",
         "react-transition-group": "^2.2.1",
-        "recharts": "^1.0.0-beta.6",
+        "recharts": "^1.0.0-beta.7",
         "style-loader": "^0.19.1",
+        "typeface-roboto": "^0.0.50",
         "url": "^0.11.0",
         "url-loader": "^0.6.2",
         "webpack": "^3.10.0"

dashboard/assets/types/content.jsx

@@ -16,38 +16,49 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-import type {ChartEntry} from './message';
-
 export type Content = {
-    home: Home,
-    chain: Chain,
-    txpool: TxPool,
-    network: Network,
-    system: System,
-    logs: Logs,
+	general: General,
+	home: Home,
+	chain: Chain,
+	txpool: TxPool,
+	network: Network,
+	system: System,
+	logs: Logs,
+};
+
+export type General = {
+	version: ?string,
+	commit: ?string,
 };
 
 export type Home = {
-    memory: Array<ChartEntry>,
-    traffic: Array<ChartEntry>,
+	memory: ChartEntries,
+	traffic: ChartEntries,
+};
+
+export type ChartEntries = Array<ChartEntry>;
+
+export type ChartEntry = {
+	time: Date,
+	value: number,
 };
 
 export type Chain = {
-    /* TODO (kurkomisi) */
+	/* TODO (kurkomisi) */
 };
 
 export type TxPool = {
-    /* TODO (kurkomisi) */
+	/* TODO (kurkomisi) */
 };
 
 export type Network = {
-    /* TODO (kurkomisi) */
+	/* TODO (kurkomisi) */
 };
 
 export type System = {
-    /* TODO (kurkomisi) */
+	/* TODO (kurkomisi) */
 };
 
 export type Logs = {
-    log: Array<string>,
+	log: Array<string>,
 };

dashboard/assets/webpack.config.js

@@ -23,7 +23,7 @@ module.exports = {
 	},
 	entry:  './index',
 	output: {
-		path:     path.resolve(__dirname, 'public'),
+		path:     path.resolve(__dirname, ''),
 		filename: 'bundle.js',
 	},
 	plugins: [

dashboard/dashboard.go

@@ -18,8 +18,9 @@ package dashboard
 
 //go:generate npm --prefix ./assets install
 //go:generate ./assets/node_modules/.bin/webpack --config ./assets/webpack.config.js --context ./assets
-//go:generate go-bindata -nometadata -o assets.go -prefix assets -nocompress -pkg dashboard assets/public/...
-//go:generate sh -c "sed 's#var _public#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go"
+//go:generate go-bindata -nometadata -o assets.go -prefix assets -nocompress -pkg dashboard assets/dashboard.html assets/bundle.js
+//go:generate sh -c "sed 's#var _bundleJs#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go"
+//go:generate sh -c "sed 's#var _dashboardHtml#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go"
 //go:generate gofmt -w -s assets.go
 
 import (
@@ -34,6 +35,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/rcrowley/go-metrics"
 	"golang.org/x/net/websocket"
@@ -53,6 +55,7 @@ type Dashboard struct {
 	listener net.Listener
 	conns    map[uint32]*client // Currently live websocket connections
 	charts   *HomeMessage
+	commit   string
 	lock     sync.RWMutex // Lock protecting the dashboard's internals
 
 	quit chan chan error // Channel used for graceful exit
@@ -67,15 +70,16 @@ type client struct {
 }
 
 // New creates a new dashboard instance with the given configuration.
-func New(config *Config) (*Dashboard, error) {
+func New(config *Config, commit string) (*Dashboard, error) {
 	return &Dashboard{
 		conns:  make(map[uint32]*client),
 		config: config,
 		quit:   make(chan chan error),
 		charts: &HomeMessage{
-			Memory:  &Chart{},
-			Traffic: &Chart{},
+			Memory:  ChartEntries{},
+			Traffic: ChartEntries{},
 		},
+		commit: commit,
 	}, nil
 }
 
@@ -87,6 +91,8 @@ func (db *Dashboard) APIs() []rpc.API { return nil }
 
 // Start implements node.Service, starting the data collection thread and the listening server of the dashboard.
 func (db *Dashboard) Start(server *p2p.Server) error {
+	log.Info("Starting dashboard")
+
 	db.wg.Add(2)
 	go db.collectData()
 	go db.collectLogs() // In case of removing this line change 2 back to 1 in wg.Add.
@@ -160,7 +166,7 @@ func (db *Dashboard) webHandler(w http.ResponseWriter, r *http.Request) {
 		w.Write(blob)
 		return
 	}
-	blob, err := Asset(filepath.Join("public", path))
+	blob, err := Asset(path[1:])
 	if err != nil {
 		log.Warn("Failed to load the asset", "path", path, "err", err)
 		http.Error(w, "not found", http.StatusNotFound)
@@ -197,15 +203,20 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) {
 			}
 		}
 	}()
+
+	versionMeta := ""
+	if len(params.VersionMeta) > 0 {
+		versionMeta = fmt.Sprintf(" (%s)", params.VersionMeta)
+	}
 	// Send the past data.
 	client.msg <- Message{
+		General: &GeneralMessage{
+			Version: fmt.Sprintf("v%d.%d.%d%s", params.VersionMajor, params.VersionMinor, params.VersionPatch, versionMeta),
+			Commit:  db.commit,
+		},
 		Home: &HomeMessage{
-			Memory: &Chart{
-				History: db.charts.Memory.History,
-			},
-			Traffic: &Chart{
-				History: db.charts.Traffic.History,
-			},
+			Memory:  db.charts.Memory,
+			Traffic: db.charts.Traffic,
 		},
 	}
 	// Start tracking the connection and drop at connection loss.
@@ -249,24 +260,20 @@ func (db *Dashboard) collectData() {
 				Value: inboundTraffic,
 			}
 			first := 0
-			if len(db.charts.Memory.History) == memorySampleLimit {
+			if len(db.charts.Memory) == memorySampleLimit {
 				first = 1
 			}
-			db.charts.Memory.History = append(db.charts.Memory.History[first:], memory)
+			db.charts.Memory = append(db.charts.Memory[first:], memory)
 			first = 0
-			if len(db.charts.Traffic.History) == trafficSampleLimit {
+			if len(db.charts.Traffic) == trafficSampleLimit {
 				first = 1
 			}
-			db.charts.Traffic.History = append(db.charts.Traffic.History[first:], traffic)
+			db.charts.Traffic = append(db.charts.Traffic[first:], traffic)
 
 			db.sendToAll(&Message{
 				Home: &HomeMessage{
-					Memory: &Chart{
-						New: memory,
-					},
-					Traffic: &Chart{
-						New: traffic,
-					},
+					Memory:  ChartEntries{memory},
+					Traffic: ChartEntries{traffic},
 				},
 			})
 		}
@@ -287,7 +294,7 @@ func (db *Dashboard) collectLogs() {
 		case <-time.After(db.config.Refresh / 2):
 			db.sendToAll(&Message{
 				Logs: &LogsMessage{
-					Log: fmt.Sprintf("%-4d: This is a fake log.", id),
+					Log: []string{fmt.Sprintf("%-4d: This is a fake log.", id)},
 				},
 			})
 			id++

dashboard/message.go

@@ -19,6 +19,7 @@ package dashboard
 import "time"
 
 type Message struct {
+	General *GeneralMessage `json:"general,omitempty"`
 	Home    *HomeMessage    `json:"home,omitempty"`
 	Chain   *ChainMessage   `json:"chain,omitempty"`
 	TxPool  *TxPoolMessage  `json:"txpool,omitempty"`
@@ -27,16 +28,18 @@ type Message struct {
 	Logs    *LogsMessage    `json:"logs,omitempty"`
 }
 
-type HomeMessage struct {
-	Memory  *Chart `json:"memory,omitempty"`
-	Traffic *Chart `json:"traffic,omitempty"`
+type GeneralMessage struct {
+	Version string `json:"version,omitempty"`
+	Commit  string `json:"commit,omitempty"`
 }
 
-type Chart struct {
-	History []*ChartEntry `json:"history,omitempty"`
-	New     *ChartEntry   `json:"new,omitempty"`
+type HomeMessage struct {
+	Memory  ChartEntries `json:"memory,omitempty"`
+	Traffic ChartEntries `json:"traffic,omitempty"`
 }
 
+type ChartEntries []*ChartEntry
+
 type ChartEntry struct {
 	Time  time.Time `json:"time,omitempty"`
 	Value float64   `json:"value,omitempty"`
@@ -59,5 +62,5 @@ type SystemMessage struct {
 }
 
 type LogsMessage struct {
-	Log string `json:"log,omitempty"`
+	Log []string `json:"log,omitempty"`
 }

ethstats/ethstats.go

@@ -613,6 +613,7 @@ func (s *Service) reportHistory(conn *websocket.Conn, list []uint64) error {
 		}
 		// Ran out of blocks, cut the report short and send
 		history = history[len(history)-i:]
+		break
 	}
 	// Assemble the history report and send it to the server
 	if len(history) > 0 {

swarm/storage/dbstore.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 )
@@ -85,6 +86,15 @@ type DbStore struct {
 	batch    *leveldb.Batch
 	lock     sync.RWMutex
 	trusted  bool // if hash integity check is to be performed (for testing only)
+
+	// Functions encodeDataFunc is used to bypass
+	// the default functionality of DbStore with
+	// mock.NodeStore for testing purposes.
+	encodeDataFunc func(chunk *Chunk) []byte
+	// If getDataFunc is defined, it will be used for
+	// retrieving the chunk data instead from the local
+	// LevelDB database.
+	getDataFunc func(key Key) (data []byte, err error)
 }
 
 // TODO: Instead of passing the distance function, just pass the address from which distances are calculated
@@ -99,6 +109,8 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uin
 	s.batchesC = make(chan struct{}, 1)
 	go s.writeBatches()
 	s.batch = new(leveldb.Batch)
+	// associate encodeData with default functionality
+	s.encodeDataFunc = encodeData
 
 	s.db, err = NewLDBDatabase(path)
 	if err != nil {
@@ -138,6 +150,22 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uin
 	return s, nil
 }
 
+// NewMockDbStore creates a new instance of DbStore with
+// mockStore set to a provided value. If mockStore argument is nil,
+// this function behaves exactly as NewDbStore.
+func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8, mockStore *mock.NodeStore) (s *DbStore, err error) {
+	s, err = NewDbStore(path, hash, capacity, po)
+	if err != nil {
+		return nil, err
+	}
+	// replace put and get with mock store functionality
+	if mockStore != nil {
+		s.encodeDataFunc = newMockEncodeDataFunc(mockStore)
+		s.getDataFunc = newMockGetDataFunc(mockStore)
+	}
+	return
+}
+
 type dpaDBIndex struct {
 	Idx    uint64
 	Access uint64
@@ -539,7 +567,7 @@ func (s *DbStore) Put(chunk *Chunk) {
 
 // force putting into db, does not check access index
 func (s *DbStore) doPut(chunk *Chunk, ikey []byte, index *dpaDBIndex, po uint8) {
-	data := encodeData(chunk)
+	data := s.encodeDataFunc(chunk)
 	s.batch.Put(getDataKey(s.dataIdx, po), data)
 	index.Idx = s.dataIdx
 	s.bucketCnt[po] = s.dataIdx
@@ -595,6 +623,19 @@ func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint
 	log.Trace(fmt.Sprintf("DbStore: batch write (%d chunks) complete", l))
 }
 
+// newMockEncodeDataFunc returns a function that stores the chunk data
+// to a mock store to bypass the default functionality encodeData.
+// The constructed function always returns the nil data, as DbStore does
+// not need to store the data, but still need to create the index.
+func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk *Chunk) []byte {
+	return func(chunk *Chunk) []byte {
+		if err := mockStore.Put(chunk.Key, chunk.SData); err != nil {
+			log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Key.Log(), err))
+		}
+		return nil
+	}
+}
+
 // try to find index; if found, update access cnt and return true
 func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool {
 	idata, err := s.db.Get(ikey)
@@ -621,15 +662,23 @@ func (s *DbStore) get(key Key) (chunk *Chunk, err error) {
 
 	if s.tryAccessIdx(getIndexKey(key), &indx) {
 		var data []byte
-
-		proximity := s.po(key)
-		datakey := getDataKey(indx.Idx, proximity)
-		data, err = s.db.Get(datakey)
-		log.Trace(fmt.Sprintf("DBStore: Chunk %v indexkey %v datakey %x proximity %d", key.Log(), indx.Idx, datakey, proximity))
-		if err != nil {
-			log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err))
-			s.delete(indx.Idx, getIndexKey(key), s.po(key))
-			return
+		if s.getDataFunc != nil {
+			// if getDataFunc is defined, use it to retrieve the chunk data
+			data, err = s.getDataFunc(key)
+			if err != nil {
+				return
+			}
+		} else {
+			// default DbStore functionality to retrieve chunk data
+			proximity := s.po(key)
+			datakey := getDataKey(indx.Idx, proximity)
+			data, err = s.db.Get(datakey)
+			log.Trace(fmt.Sprintf("DBStore: Chunk %v indexkey %v datakey %x proximity %d", key.Log(), indx.Idx, datakey, proximity))
+			if err != nil {
+				log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err))
+				s.delete(indx.Idx, getIndexKey(key), s.po(key))
+				return
+			}
 		}
 
 		if !s.trusted {
@@ -647,12 +696,26 @@ func (s *DbStore) get(key Key) (chunk *Chunk, err error) {
 
 		chunk = NewChunk(key, nil)
 		decodeData(data, chunk)
+
 	} else {
 		err = notFound
 	}
 
 	return
+}
 
+// newMockGetFunc returns a function that reads chunk data from
+// the mock database, which is used as the value for DbStore.getFunc
+// to bypass the default functionality of DbStore with a mock store.
+func newMockGetDataFunc(mockStore *mock.NodeStore) func(key Key) (data []byte, err error) {
+	return func(key Key) (data []byte, err error) {
+		data, err = mockStore.Get(key)
+		if err == mock.ErrNotFound {
+			// preserve notFound error
+			err = notFound
+		}
+		return data, err
+	}
 }
 
 func (s *DbStore) updateAccessCnt(key Key) {

swarm/storage/dbstore_test.go

@@ -21,10 +21,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 )
 
 type testDbStore struct {
@@ -37,12 +41,16 @@ func newTestDbStore() (*testDbStore, error) {
 	if err != nil {
 		return nil, err
 	}
-	basekey := make([]byte, 32)
-	db, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
+	db, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc)
 
 	return &testDbStore{db, dir}, err
 }
 
+func testPoFunc(k Key) (ret uint8) {
+	basekey := make([]byte, 32)
+	return uint8(Proximity(basekey[:], k[:]))
+}
+
 func (db *testDbStore) close() {
 	db.Close()
 	err := os.RemoveAll(db.dir)
@@ -201,3 +209,77 @@ func BenchmarkDbStoreGet_1_5k(b *testing.B) {
 func BenchmarkDbStoreGet_8_5k(b *testing.B) {
 	benchmarkDbStoreGet(5000, 8, 4096, b)
 }
+
+func initMockDbStore(t *testing.T, mockStore *mock.NodeStore) *DbStore {
+	dir, err := ioutil.TempDir("", "bzz-storage-test-mock")
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc, mockStore)
+	if err != nil {
+		t.Fatal("can't create store:", err)
+	}
+	return m
+}
+
+// testMockDbStore runs the same tests as testDbStore but with mock store configured.
+// It also verifies if mock global store is storing the chunk data.
+func testMockDbStore(l int64, branches int64, t *testing.T) {
+	globalStore := mem.NewGlobalStore()
+	addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
+	mockStore := globalStore.NewNodeStore(addr)
+	m := initMockDbStore(t, mockStore)
+	defer m.Close()
+
+	key := Key(common.Hex2Bytes("fed1911825fc6a02ebfd19ab218a20455d8d7d275f8bf4d8244eb04364fae6f7"))
+	data := common.Hex2BytesFixed(strings.Repeat("1234567890abcdf", 10), 4096)
+
+	m.Put(&Chunk{
+		Key:   key,
+		SData: data,
+	})
+
+	_, err := globalStore.Get(addr, key)
+	if err != nil {
+		t.Errorf("unexpected error getting the data from global mock store: %v", err)
+	}
+
+	if !globalStore.HasKey(addr, key) {
+		t.Error("key not found in global store")
+	}
+
+	// TODO: fix this!
+	// testStoreRandom(m, 8, l, chunk.S, t)
+
+}
+
+func TestMockDbStore128_0x1000000(t *testing.T) {
+	testMockDbStore(0x1000000, 128, t)
+}
+
+func TestMockDbStore128_10000_(t *testing.T) {
+	testMockDbStore(10000, 128, t)
+}
+
+func TestMockDbStore128_1000_(t *testing.T) {
+	testMockDbStore(1000, 128, t)
+}
+
+func TestMockDbStore128_100_(t *testing.T) {
+	testMockDbStore(100, 128, t)
+}
+
+func TestMockDbStore2_100_(t *testing.T) {
+	testMockDbStore(100, 2, t)
+}
+
+func TestMockDbStoreNotFound(t *testing.T) {
+	globalStore := mem.NewGlobalStore()
+	mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed"))
+	m := initMockDbStore(t, mockStore)
+	defer m.Close()
+	_, err := m.Get(ZeroKey)
+	if err != notFound {
+		t.Errorf("Expected notFound, got %v", err)
+	}
+}

swarm/storage/localstore.go

@@ -56,23 +56,9 @@ func NewTestLocalStore(path string) (*LocalStore, error) {
 	return localStore, nil
 }
 
-func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) {
-	hasher := MakeHashFunc("SHA3")
-	dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
-	if err != nil {
-		return nil, err
-	}
-	localStore := &LocalStore{
-		memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
-		DbStore:  dbStore,
-	}
-	return localStore, nil
-}
-
 // LocalStore is itself a chunk store
 // unsafe, in that the data is not integrity checked
 func (self *LocalStore) Put(chunk *Chunk) {
-	chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
 	self.memStore.Put(chunk)
 	go func() {
 		self.DbStore.Put(chunk)

swarm/storage/mock/db/db.go

@@ -0,0 +1,236 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package db implements a mock store that keeps all chunk data in LevelDB database.
+package db
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore contains the LevelDB database that is storing
+// chunk data for all swarm nodes.
+// Closing the GlobalStore with Close method is required to
+// release resources used by the database.
+type GlobalStore struct {
+	db *leveldb.DB
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore(path string) (s *GlobalStore, err error) {
+	db, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &GlobalStore{
+		db: db,
+	}, nil
+}
+
+// Close releases the resources used by the underlying LevelDB.
+func (s *GlobalStore) Close() error {
+	return s.db.Close()
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore {
+	return mock.NewNodeStore(addr, s)
+}
+
+// Get returns chunk data if the chunk with key exists for node
+// on address addr.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	has, err := s.db.Has(nodeDBKey(addr, key), nil)
+	if err != nil {
+		return nil, mock.ErrNotFound
+	}
+	if !has {
+		return nil, mock.ErrNotFound
+	}
+	data, err = s.db.Get(dataDBKey(key), nil)
+	if err == leveldb.ErrNotFound {
+		err = mock.ErrNotFound
+	}
+	return
+}
+
+// Put saves the chunk data for node with address addr.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	batch := new(leveldb.Batch)
+	batch.Put(nodeDBKey(addr, key), nil)
+	batch.Put(dataDBKey(key), data)
+	return s.db.Write(batch, nil)
+}
+
+// HasKey returns whether a node with addr contains the key.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	has, err := s.db.Has(nodeDBKey(addr, key), nil)
+	if err != nil {
+		has = false
+	}
+	return has
+}
+
+// Import reads tar archive from a reader that contains exported chunk data.
+// It returns the number of chunks imported and an error.
+func (s *GlobalStore) Import(r io.Reader) (n int, err error) {
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return n, err
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return n, err
+		}
+
+		var c mock.ExportedChunk
+		if err = json.Unmarshal(data, &c); err != nil {
+			return n, err
+		}
+
+		batch := new(leveldb.Batch)
+		for _, addr := range c.Addrs {
+			batch.Put(nodeDBKeyHex(addr, hdr.Name), nil)
+		}
+
+		batch.Put(dataDBKey(common.Hex2Bytes(hdr.Name)), c.Data)
+		if err = s.db.Write(batch, nil); err != nil {
+			return n, err
+		}
+
+		n++
+	}
+	return n, err
+}
+
+// Export writes to a writer a tar archive with all chunk data from
+// the store. It returns the number fo chunks exported and an error.
+func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	encoder := json.NewEncoder(buf)
+
+	iter := s.db.NewIterator(util.BytesPrefix(nodeKeyPrefix), nil)
+	defer iter.Release()
+
+	var currentKey string
+	var addrs []common.Address
+
+	saveChunk := func(hexKey string) error {
+		key := common.Hex2Bytes(hexKey)
+
+		data, err := s.db.Get(dataDBKey(key), nil)
+		if err != nil {
+			return err
+		}
+
+		buf.Reset()
+		if err = encoder.Encode(mock.ExportedChunk{
+			Addrs: addrs,
+			Data:  data,
+		}); err != nil {
+			return err
+		}
+
+		d := buf.Bytes()
+		hdr := &tar.Header{
+			Name: hexKey,
+			Mode: 0644,
+			Size: int64(len(d)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if _, err := tw.Write(d); err != nil {
+			return err
+		}
+		n++
+		return nil
+	}
+
+	for iter.Next() {
+		k := bytes.TrimPrefix(iter.Key(), nodeKeyPrefix)
+		i := bytes.Index(k, []byte("-"))
+		if i < 0 {
+			continue
+		}
+		hexKey := string(k[:i])
+
+		if currentKey == "" {
+			currentKey = hexKey
+		}
+
+		if hexKey != currentKey {
+			if err = saveChunk(currentKey); err != nil {
+				return n, err
+			}
+
+			addrs = addrs[:0]
+		}
+
+		currentKey = hexKey
+		addrs = append(addrs, common.BytesToAddress(k[i:]))
+	}
+
+	if len(addrs) > 0 {
+		if err = saveChunk(currentKey); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+var (
+	nodeKeyPrefix = []byte("node-")
+	dataKeyPrefix = []byte("data-")
+)
+
+// nodeDBKey constructs a database key for key/node mappings.
+func nodeDBKey(addr common.Address, key []byte) []byte {
+	return nodeDBKeyHex(addr, common.Bytes2Hex(key))
+}
+
+// nodeDBKeyHex constructs a database key for key/node mappings
+// using the hexadecimal string representation of the key.
+func nodeDBKeyHex(addr common.Address, hexKey string) []byte {
+	return append(append(nodeKeyPrefix, []byte(hexKey+"-")...), addr[:]...)
+}
+
+// dataDBkey constructs a database key for key/data storage.
+func dataDBKey(key []byte) []byte {
+	return append(dataKeyPrefix, key...)
+}

swarm/storage/mock/db/db_test.go

@@ -0,0 +1,73 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package db
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+// TestDBStore is running a test.MockStore tests
+// using test.MockStore function.
+func TestDBStore(t *testing.T) {
+	dir, err := ioutil.TempDir("", "mock_"+t.Name())
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir)
+
+	store, err := NewGlobalStore(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+
+	test.MockStore(t, store, 100)
+}
+
+// TestImportExport is running a test.ImportExport tests
+// using test.MockStore function.
+func TestImportExport(t *testing.T) {
+	dir1, err := ioutil.TempDir("", "mock_"+t.Name()+"_exporter")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir1)
+
+	store1, err := NewGlobalStore(dir1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store1.Close()
+
+	dir2, err := ioutil.TempDir("", "mock_"+t.Name()+"_importer")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir2)
+
+	store2, err := NewGlobalStore(dir2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store2.Close()
+
+	test.ImportExport(t, store1, store2, 100)
+}

swarm/storage/mock/mem/mem.go

@@ -0,0 +1,175 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mem implements a mock store that keeps all chunk data in memory.
+// While it can be used for testing on smaller scales, the main purpose of this
+// package is to provide the simplest reference implementation of a mock store.
+package mem
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore stores all chunk data and also keys and node addresses relations.
+// It implements mock.GlobalStore interface.
+type GlobalStore struct {
+	nodes map[string]map[common.Address]struct{}
+	data  map[string][]byte
+	mu    sync.Mutex
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore() *GlobalStore {
+	return &GlobalStore{
+		nodes: make(map[string]map[common.Address]struct{}),
+		data:  make(map[string][]byte),
+	}
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore {
+	return mock.NewNodeStore(addr, s)
+}
+
+// Get returns chunk data if the chunk with key exists for node
+// on address addr.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.nodes[string(key)][addr]; !ok {
+		return nil, mock.ErrNotFound
+	}
+
+	data, ok := s.data[string(key)]
+	if !ok {
+		return nil, mock.ErrNotFound
+	}
+	return data, nil
+}
+
+// Put saves the chunk data for node with address addr.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.nodes[string(key)]; !ok {
+		s.nodes[string(key)] = make(map[common.Address]struct{})
+	}
+	s.nodes[string(key)][addr] = struct{}{}
+	s.data[string(key)] = data
+	return nil
+}
+
+// HasKey returns whether a node with addr contains the key.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	_, ok := s.nodes[string(key)][addr]
+	return ok
+}
+
+// Import reads tar archive from a reader that contains exported chunk data.
+// It returns the number of chunks imported and an error.
+func (s *GlobalStore) Import(r io.Reader) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return n, err
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return n, err
+		}
+
+		var c mock.ExportedChunk
+		if err = json.Unmarshal(data, &c); err != nil {
+			return n, err
+		}
+
+		addrs := make(map[common.Address]struct{})
+		for _, a := range c.Addrs {
+			addrs[a] = struct{}{}
+		}
+
+		key := string(common.Hex2Bytes(hdr.Name))
+		s.nodes[key] = addrs
+		s.data[key] = c.Data
+		n++
+	}
+	return n, err
+}
+
+// Export writes to a writer a tar archive with all chunk data from
+// the store. It returns the number of chunks exported and an error.
+func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	encoder := json.NewEncoder(buf)
+	for key, addrs := range s.nodes {
+		al := make([]common.Address, 0, len(addrs))
+		for a := range addrs {
+			al = append(al, a)
+		}
+
+		buf.Reset()
+		if err = encoder.Encode(mock.ExportedChunk{
+			Addrs: al,
+			Data:  s.data[key],
+		}); err != nil {
+			return n, err
+		}
+
+		data := buf.Bytes()
+		hdr := &tar.Header{
+			Name: common.Bytes2Hex([]byte(key)),
+			Mode: 0644,
+			Size: int64(len(data)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return n, err
+		}
+		if _, err := tw.Write(data); err != nil {
+			return n, err
+		}
+		n++
+	}
+	return n, err
+}

swarm/storage/mock/mem/mem_test.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mem
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+// TestDBStore is running test for a GlobalStore
+// using test.MockStore function.
+func TestMemStore(t *testing.T) {
+	test.MockStore(t, NewGlobalStore(), 100)
+}
+
+// TestImportExport is running tests for importing and
+// exporting data between two GlobalStores
+// using test.ImportExport function.
+func TestImportExport(t *testing.T) {
+	test.ImportExport(t, NewGlobalStore(), NewGlobalStore(), 100)
+}

swarm/storage/mock/mock.go

@@ -0,0 +1,111 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mock defines types that are used by different implementations
+// of mock storages.
+//
+// Implementations of mock storages are located in directories
+// under this package:
+//
+//  - db - LevelDB backend
+//  - mem - in memory map backend
+//  - rpc - RPC client that can connect to other backends
+//
+// Mock storages can implement Importer and Exporter interfaces
+// for importing and exporting all chunk data that they contain.
+// The exported file is a tar archive with all files named by
+// hexadecimal representations of chunk keys and with content
+// with JSON-encoded ExportedChunk structure. Exported format
+// should be preserved across all mock store implementations.
+package mock
+
+import (
+	"errors"
+	"io"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// ErrNotFound indicates that the chunk is not found.
+var ErrNotFound = errors.New("not found")
+
+// NodeStore holds the node address and a reference to the GlobalStore
+// in order to access and store chunk data only for one node.
+type NodeStore struct {
+	store GlobalStorer
+	addr  common.Address
+}
+
+// NewNodeStore creates a new instance of NodeStore that keeps
+// chunk data using GlobalStorer with a provided address.
+func NewNodeStore(addr common.Address, store GlobalStorer) *NodeStore {
+	return &NodeStore{
+		store: store,
+		addr:  addr,
+	}
+}
+
+// Get returns chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Get(key []byte) (data []byte, err error) {
+	return n.store.Get(n.addr, key)
+}
+
+// Put saves chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Put(key []byte, data []byte) error {
+	return n.store.Put(n.addr, key, data)
+}
+
+// GlobalStorer defines methods for mock db store
+// that stores chunk data for all swarm nodes.
+// It is used in tests to construct mock NodeStores
+// for swarm nodes and to track and validate chunks.
+type GlobalStorer interface {
+	Get(addr common.Address, key []byte) (data []byte, err error)
+	Put(addr common.Address, key []byte, data []byte) error
+	HasKey(addr common.Address, key []byte) bool
+	// NewNodeStore creates an instance of NodeStore
+	// to be used by a single swarm node with
+	// address addr.
+	NewNodeStore(addr common.Address) *NodeStore
+}
+
+// Importer defines method for importing mock store data
+// from an exported tar archive.
+type Importer interface {
+	Import(r io.Reader) (n int, err error)
+}
+
+// Exporter defines method for exporting mock store data
+// to a tar archive.
+type Exporter interface {
+	Export(w io.Writer) (n int, err error)
+}
+
+// ImportExporter is an interface for importing and exporting
+// mock store data to and from a tar archive.
+type ImportExporter interface {
+	Importer
+	Exporter
+}
+
+// ExportedChunk is the structure that is saved in tar archive for
+// each chunk as JSON-encoded bytes.
+type ExportedChunk struct {
+	Data  []byte           `json:"d"`
+	Addrs []common.Address `json:"a"`
+}

swarm/storage/mock/rpc/rpc.go

@@ -0,0 +1,84 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package rpc implements an RPC client that connect to a centralized mock store.
+// Centralazied mock store can be any other mock store implementation that is
+// registered to Ethereum RPC server under mockStore name. Methods that defines
+// mock.GlobalStore are the same that are used by RPC. Example:
+//
+//   server := rpc.NewServer()
+//   server.RegisterName("mockStore", mem.NewGlobalStore())
+package rpc
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore is rpc.Client that connects to a centralized mock store.
+// Closing GlobalStore instance is required to release RPC client resources.
+type GlobalStore struct {
+	client *rpc.Client
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore(client *rpc.Client) *GlobalStore {
+	return &GlobalStore{
+		client: client,
+	}
+}
+
+// Close closes RPC client.
+func (s *GlobalStore) Close() error {
+	s.client.Close()
+	return nil
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore {
+	return mock.NewNodeStore(addr, s)
+}
+
+// Get calls a Get method to RPC server.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	err = s.client.Call(&data, "mockStore_get", addr, key)
+	if err != nil && err.Error() == "not found" {
+		// pass the mock package value of error instead an rpc error
+		return data, mock.ErrNotFound
+	}
+	return data, err
+}
+
+// Put calls a Put method to RPC server.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	err := s.client.Call(nil, "mockStore_put", addr, key, data)
+	return err
+}
+
+// HasKey calls a HasKey method to RPC server.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	var has bool
+	if err := s.client.Call(&has, "mockStore_hasKey", addr, key); err != nil {
+		log.Error(fmt.Sprintf("mock store HasKey: addr %s, key %064x: %v", addr, key, err))
+		return false
+	}
+	return has
+}

swarm/storage/mock/rpc/rpc_test.go

@@ -1,6 +1,4 @@
-// @flow
-
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -16,46 +14,28 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-export type Message = {
-    home?: HomeMessage,
-    chain?: ChainMessage,
-    txpool?: TxPoolMessage,
-    network?: NetworkMessage,
-    system?: SystemMessage,
-    logs?: LogsMessage,
-};
+package rpc
 
-export type HomeMessage = {
-    memory?: Chart,
-    traffic?: Chart,
-};
+import (
+	"testing"
 
-export type Chart = {
-    history?: Array<ChartEntry>,
-    new?: ChartEntry,
-};
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
 
-export type ChartEntry = {
-    time: Date,
-    value: number,
-};
+// TestDBStore is running test for a GlobalStore
+// using test.MockStore function.
+func TestRPCStore(t *testing.T) {
+	serverStore := mem.NewGlobalStore()
 
-export type ChainMessage = {
-    /* TODO (kurkomisi) */
-};
+	server := rpc.NewServer()
+	if err := server.RegisterName("mockStore", serverStore); err != nil {
+		t.Fatal(err)
+	}
 
-export type TxPoolMessage = {
-    /* TODO (kurkomisi) */
-};
+	store := NewGlobalStore(rpc.DialInProc(server))
+	defer store.Close()
 
-export type NetworkMessage = {
-    /* TODO (kurkomisi) */
-};
-
-export type SystemMessage = {
-    /* TODO (kurkomisi) */
-};
-
-export type LogsMessage = {
-    log: string,
-};
+	test.MockStore(t, store, 100)
+}

swarm/storage/mock/test/test.go

@@ -0,0 +1,186 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package test provides functions that are used for testing
+// GlobalStorer implementations.
+package test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// MockStore creates NodeStore instances from provided GlobalStorer,
+// each one with a unique address, stores different chunks on them
+// and checks if they are retrievable or not on all nodes.
+// Attribute n defines the number of NodeStores that will be created.
+func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
+	t.Run("GlobalStore", func(t *testing.T) {
+		addrs := make([]common.Address, n)
+		for i := 0; i < n; i++ {
+			addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+		}
+
+		for i, addr := range addrs {
+			key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+			data := []byte(strconv.FormatInt(int64(i)+1, 16))
+			data = append(data, make([]byte, 4096-len(data))...)
+			globalStore.Put(addr, key, data)
+
+			for _, cAddr := range addrs {
+				cData, err := globalStore.Get(cAddr, key)
+				if cAddr == addr {
+					if err != nil {
+						t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+					}
+					if !bytes.Equal(data, cData) {
+						t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+					}
+					if !globalStore.HasKey(cAddr, key) {
+						t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+					}
+				} else {
+					if err != mock.ErrNotFound {
+						t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+					}
+					if len(cData) > 0 {
+						t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+					}
+					if globalStore.HasKey(cAddr, key) {
+						t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+					}
+				}
+			}
+		}
+	})
+
+	t.Run("NodeStore", func(t *testing.T) {
+		nodes := make(map[common.Address]*mock.NodeStore)
+		for i := 0; i < n; i++ {
+			addr := common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+			nodes[addr] = globalStore.NewNodeStore(addr)
+		}
+
+		i := 0
+		for addr, store := range nodes {
+			i++
+			key := storage.Key(append(addr[:], []byte(fmt.Sprintf("%x", i))...))
+			data := []byte(strconv.FormatInt(int64(i)+1, 16))
+			data = append(data, make([]byte, 4096-len(data))...)
+			store.Put(key, data)
+
+			for cAddr, cStore := range nodes {
+				cData, err := cStore.Get(key)
+				if cAddr == addr {
+					if err != nil {
+						t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+					}
+					if !bytes.Equal(data, cData) {
+						t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+					}
+					if !globalStore.HasKey(cAddr, key) {
+						t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+					}
+				} else {
+					if err != mock.ErrNotFound {
+						t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+					}
+					if len(cData) > 0 {
+						t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+					}
+					if globalStore.HasKey(cAddr, key) {
+						t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+					}
+				}
+			}
+		}
+	})
+}
+
+// ImportExport saves chunks to the outStore, exports them to the tar archive,
+// imports tar archive to the inStore and checks if all chunks are imported correctly.
+func ImportExport(t *testing.T, outStore, inStore mock.GlobalStorer, n int) {
+	exporter, ok := outStore.(mock.Exporter)
+	if !ok {
+		t.Fatal("outStore does not implement mock.Exporter")
+	}
+	importer, ok := inStore.(mock.Importer)
+	if !ok {
+		t.Fatal("inStore does not implement mock.Importer")
+	}
+	addrs := make([]common.Address, n)
+	for i := 0; i < n; i++ {
+		addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+	}
+
+	for i, addr := range addrs {
+		key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+		data := []byte(strconv.FormatInt(int64(i)+1, 16))
+		data = append(data, make([]byte, 4096-len(data))...)
+		outStore.Put(addr, key, data)
+	}
+
+	r, w := io.Pipe()
+	defer r.Close()
+
+	go func() {
+		defer w.Close()
+		if _, err := exporter.Export(w); err != nil {
+			t.Fatalf("export: %v", err)
+		}
+	}()
+
+	if _, err := importer.Import(r); err != nil {
+		t.Fatalf("import: %v", err)
+	}
+
+	for i, addr := range addrs {
+		key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+		data := []byte(strconv.FormatInt(int64(i)+1, 16))
+		data = append(data, make([]byte, 4096-len(data))...)
+		for _, cAddr := range addrs {
+			cData, err := inStore.Get(cAddr, key)
+			if cAddr == addr {
+				if err != nil {
+					t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+				}
+				if !bytes.Equal(data, cData) {
+					t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+				}
+				if !inStore.HasKey(cAddr, key) {
+					t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+				}
+			} else {
+				if err != mock.ErrNotFound {
+					t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+				}
+				if len(cData) > 0 {
+					t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+				}
+				if inStore.HasKey(cAddr, key) {
+					t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+				}
+			}
+		}
+	}
+}

swarm/storage/resource.go

@@ -11,7 +11,6 @@ import (
 	"golang.org/x/net/idna"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -26,11 +25,11 @@ const (
 // version of the resource update data.
 type resource struct {
 	name       string
-	ensName    common.Hash
+	nameHash   common.Hash
 	startBlock uint64
-	lastBlock  uint64
+	lastPeriod uint32
 	frequency  uint64
-	version    uint64
+	version    uint32
 	data       []byte
 	updated    time.Time
 }
@@ -40,10 +39,6 @@ type resource struct {
 // The update scheme is built on swarm chunks with chunk keys following
 // a predictable, versionable pattern.
 //
-// The data of the chunk contains the content hash of the version in question.
-// In order to be valid, the hash is signed by the owner of the ENS record
-// of the mutable resource.
-//
 // Updates are defined to be periodic in nature, where periods are
 // expressed in terms of number of blocks.
 //
@@ -56,7 +51,7 @@ type resource struct {
 
 // The root entry tells the requester from when the mutable resource was
 // first added (block number) and in which block number to look for the
-// actual updates. Thus, a resource update for identifier "foo.bar"
+// actual updates. Thus, a resource update for identifier "føø.bar"
 // starting at block 4200 with frequency 42 will have updates on block 4242,
 // 4284, 4326 and so on.
 //
@@ -66,27 +61,28 @@ type resource struct {
 //
 // Note that the root entry is not required for the resource update scheme to
 // work. A normal chunk of the blocknumber/frequency data can also be created,
-// and pointed to by an actual ENS entry (or manifest entry) instead.
+// and pointed to by an external resource (ENS or manifest entry)
 //
 // Actual data updates are also made in the form of swarm chunks. The keys
 // of the updates are the hash of a concatenation of properties as follows:
 //
-// sha256(namehash|blocknumber|version)
+// sha256(namehash|period|version)
 //
-// The blocknumber here is the next block period after the current block
-// calculated from the start block and frequency of the resource update.
-// Using our previous example, this means that an update made at block 4285,
-// and even 4284, will have 4326 as the block number.
+// The period is (currentblock - startblock) / frequency
+//
+// Using our previous example, this means that a period 3 will have 4326 as
+// the block number.
 //
 // If more than one update is made to the same block number, incremental
 // version numbers are used successively.
 //
 // A lookup agent need only know the identifier name in order to get the versions
 //
-// the data itself is prefixed with a signed hash of the data. The sigining key
-// is used to verify the authenticity of the update, for example by looking
-// up the ownership of the namehash in ENS and comparing to the address derived
-// from it
+// the chunk data is: sign(resourcedata)|resourcedata
+// the resourcedata is: headerlength|period|version|name|data
+//
+// headerlength is a 16 bit value containing the byte length of period|version|name
+// period and version are both 32 bit values. name can have arbitrary length
 //
 // NOTE: the following is yet to be implemented
 // The resource update chunks will be stored in the swarm, but receive special
@@ -94,29 +90,48 @@ type resource struct {
 // stored using a separate store, and forwarding/syncing protocols carry per-chunk
 // flags to tell whether the chunk can be validated or not; if not it is to be
 // treated as a resource update chunk.
-//
-// TODO: signature validation
+
+type ResourceValidator interface {
+	isOwner(string) (bool, error)
+	nameHash(string) common.Hash
+}
+
 type ResourceHandler struct {
 	ChunkStore
-	ethapi       *rpc.Client
+	validator    ResourceValidator
+	rpcClient    *rpc.Client
 	resources    map[string]*resource
 	hashLock     sync.Mutex
-	resourceLock sync.Mutex
+	resourceLock sync.RWMutex
 	hasher       SwarmHash
 	privKey      *ecdsa.PrivateKey
 	maxChunkData int64
 }
 
 // Create or open resource update chunk store
-func NewResourceHandler(privKey *ecdsa.PrivateKey, hasher SwarmHasher, chunkStore ChunkStore, ethapi *rpc.Client) (*ResourceHandler, error) {
-	return &ResourceHandler{
+func NewResourceHandler(privKey *ecdsa.PrivateKey, hasher SwarmHasher, chunkStore ChunkStore, rpcClient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) {
+	rh := &ResourceHandler{
 		ChunkStore:   chunkStore,
-		ethapi:       ethapi,
+		rpcClient:    rpcClient,
 		resources:    make(map[string]*resource),
 		hasher:       hasher(),
 		privKey:      privKey,
 		maxChunkData: DefaultBranches * int64(hasher().Size()),
-	}, nil
+	}
+
+	if validator != nil {
+		rh.validator = validator
+	} else {
+		rh.validator = NewGenericValidator(func(name string) common.Hash {
+			rh.hashLock.Lock()
+			defer rh.hashLock.Unlock()
+			rh.hasher.Reset()
+			rh.hasher.Write([]byte(name))
+			return common.BytesToHash(rh.hasher.Sum(nil))
+		})
+	}
+
+	return rh, nil
 }
 
 func validateInput(name string, frequency uint64) (string, error) {
@@ -142,7 +157,7 @@ func validateInput(name string, frequency uint64) (string, error) {
 // Creates a standalone resource object
 //
 // Can be passed to SetResource if external root data lookups are used
-func NewResource(name string, startBlock uint64, frequency uint64) (*resource, error) {
+func NewResource(name string, startBlock uint64, frequency uint64, nameHashFunc func(name string) common.Hash) (*resource, error) {
 
 	validname, err := validateInput(name, frequency)
 	if err != nil {
@@ -151,23 +166,30 @@ func NewResource(name string, startBlock uint64, frequency uint64) (*resource, e
 
 	return &resource{
 		name:       validname,
-		ensName:    ens.EnsNode(validname),
+		nameHash:   nameHashFunc(validname),
 		startBlock: startBlock,
 		frequency:  frequency,
 	}, nil
 }
 
-// Creates a new root entry for a resource update identified by `name` with the specified `frequency`.
+// Creates a new root entry for a mutable resource identified by `name` with the specified `frequency`.
 //
 // The start block of the resource update will be the actual current block height of the connected network.
 func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resource, error) {
 
+	ok, err := self.validator.isOwner(name)
+	if err != nil {
+		return nil, err
+	} else if !ok {
+		return nil, fmt.Errorf("Not owner of '%s'", name)
+	}
+
 	validname, err := validateInput(name, frequency)
 	if err != nil {
 		return nil, err
 	}
 
-	ensName := ens.EnsNode(validname)
+	nameHash := self.validator.nameHash(validname)
 
 	// get our blockheight at this time
 	currentblock, err := self.getBlock()
@@ -177,7 +199,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour
 
 	// chunk with key equal to namehash points to data of first blockheight + update frequency
 	// from this we know from what blockheight we should look for updates, and how often
-	chunk := NewChunk(Key(ensName[:]), nil)
+	chunk := NewChunk(Key(nameHash[:]), nil)
 	chunk.SData = make([]byte, indexSize)
 
 	// resource update root chunks follow same convention as "normal" chunks
@@ -189,17 +211,17 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour
 	binary.LittleEndian.PutUint64(val, frequency)
 	copy(chunk.SData[16:], val)
 	self.Put(chunk)
-	log.Debug("new resource", "name", validname, "key", ensName, "startBlock", currentblock, "frequency", frequency)
+	log.Debug("new resource", "name", validname, "key", nameHash, "startBlock", currentblock, "frequency", frequency)
 
-	self.resourceLock.Lock()
-	defer self.resourceLock.Unlock()
-	self.resources[name] = &resource{
+	rsrc := &resource{
 		name:       validname,
-		ensName:    ensName,
+		nameHash:   nameHash,
 		startBlock: currentblock,
 		frequency:  frequency,
 		updated:    time.Now(),
 	}
+	self.setResource(name, rsrc)
+
 	return self.resources[name], nil
 }
 
@@ -210,7 +232,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour
 //
 // Method will fail if resource is already registered in this session, unless
 // `allowOverwrite` is set
-func (self *ResourceHandler) SetResource(rsrc *resource, allowOverwrite bool) error {
+func (self *ResourceHandler) SetExternalResource(rsrc *resource, allowOverwrite bool) error {
 
 	utfname, err := idna.ToUnicode(rsrc.name)
 	if err != nil {
@@ -247,12 +269,12 @@ func (self *ResourceHandler) SetResource(rsrc *resource, allowOverwrite bool) er
 // root chunk.
 // It is the callers responsibility to make sure that this chunk exists (if the resource
 // update root data was retrieved externally, it typically doesn't)
-func (self *ResourceHandler) LookupVersion(name string, nextblock uint64, version uint64, refresh bool) (*resource, error) {
+func (self *ResourceHandler) LookupVersion(name string, period uint32, version uint32, refresh bool) (*resource, error) {
 	rsrc, err := self.loadResource(name, refresh)
 	if err != nil {
 		return nil, err
 	}
-	return self.lookup(rsrc, name, nextblock, version, refresh)
+	return self.lookup(rsrc, name, period, version, refresh)
 }
 
 // Retrieves the latest version of the resource update identified by `name`
@@ -263,12 +285,12 @@ func (self *ResourceHandler) LookupVersion(name string, nextblock uint64, versio
 // and returned.
 //
 // See also (*ResourceHandler).LookupVersion
-func (self *ResourceHandler) LookupHistorical(name string, nextblock uint64, refresh bool) (*resource, error) {
+func (self *ResourceHandler) LookupHistorical(name string, period uint32, refresh bool) (*resource, error) {
 	rsrc, err := self.loadResource(name, refresh)
 	if err != nil {
 		return nil, err
 	}
-	return self.lookup(rsrc, name, nextblock, 0, refresh)
+	return self.lookup(rsrc, name, period, 0, refresh)
 }
 
 // Retrieves the latest version of the resource update identified by `name`
@@ -292,15 +314,15 @@ func (self *ResourceHandler) LookupLatest(name string, refresh bool) (*resource,
 	if err != nil {
 		return nil, err
 	}
-	nextblock := getNextBlock(rsrc.startBlock, currentblock, rsrc.frequency)
-	return self.lookup(rsrc, name, nextblock, 0, refresh)
+	nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency)
+	return self.lookup(rsrc, name, nextperiod, 0, refresh)
 }
 
 // base code for public lookup methods
-func (self *ResourceHandler) lookup(rsrc *resource, name string, nextblock uint64, version uint64, refresh bool) (*resource, error) {
+func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, version uint32, refresh bool) (*resource, error) {
 
-	if nextblock == 0 {
-		return nil, fmt.Errorf("blocknumber must be >0")
+	if period == 0 {
+		return nil, fmt.Errorf("period must be >0")
 	}
 
 	// start from the last possible block period, and iterate previous ones until we find a match
@@ -312,29 +334,29 @@ func (self *ResourceHandler) lookup(rsrc *resource, name string, nextblock uint6
 		version = 1
 	}
 
-	for nextblock > rsrc.startBlock {
-		key := self.resourceHash(rsrc.ensName, nextblock, version)
+	for period > 0 {
+		key := self.resourceHash(rsrc.nameHash, period, version)
 		chunk, err := self.Get(key)
 		if err == nil {
 			if specificversion {
-				return self.updateResourceIndex(rsrc, chunk, nextblock, version, &name)
+				return self.updateResourceIndex(rsrc, chunk, &name)
 			}
 			// check if we have versions > 1. If a version fails, the previous version is used and returned.
-			log.Trace("rsrc update version 1 found, checking for version updates", "nextblock", nextblock, "key", key)
+			log.Trace("rsrc update version 1 found, checking for version updates", "period", period, "key", key)
 			for {
 				newversion := version + 1
-				key := self.resourceHash(rsrc.ensName, nextblock, newversion)
+				key := self.resourceHash(rsrc.nameHash, period, newversion)
 				newchunk, err := self.Get(key)
 				if err != nil {
-					return self.updateResourceIndex(rsrc, chunk, nextblock, version, &name)
+					return self.updateResourceIndex(rsrc, chunk, &name)
 				}
-				log.Trace("version update found, checking next", "version", version, "block", nextblock, "key", key)
+				log.Trace("version update found, checking next", "version", version, "period", period, "key", key)
 				chunk = newchunk
 				version = newversion
 			}
 		}
-		log.Trace("rsrc update not found, checking previous period", "block", nextblock, "key", key)
-		nextblock -= rsrc.frequency
+		log.Trace("rsrc update not found, checking previous period", "period", period, "key", key)
+		period--
 	}
 	return nil, fmt.Errorf("no updates found")
 }
@@ -344,22 +366,19 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource,
 	// if the resource is not known to this session we must load it
 	// if refresh is set, we force load
 
-	rsrc := &resource{}
-
-	self.resourceLock.Lock()
-	_, ok := self.resources[name]
-	self.resourceLock.Unlock()
-	if !ok || refresh {
+	rsrc := self.getResource(name)
+	if rsrc == nil || refresh {
+		rsrc = &resource{}
 		// make sure our ens identifier is idna safe
 		validname, err := idna.ToASCII(name)
 		if err != nil {
 			return nil, err
 		}
 		rsrc.name = validname
-		rsrc.ensName = ens.EnsNode(validname)
+		rsrc.nameHash = self.validator.nameHash(validname)
 
 		// get the root info chunk and update the cached value
-		chunk, err := self.Get(Key(rsrc.ensName[:]))
+		chunk, err := self.Get(Key(rsrc.nameHash[:]))
 		if err != nil {
 			return nil, err
 		}
@@ -378,7 +397,7 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource,
 		rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[16:])
 	} else {
 		rsrc.name = self.resources[name].name
-		rsrc.ensName = self.resources[name].ensName
+		rsrc.nameHash = self.resources[name].nameHash
 		rsrc.startBlock = self.resources[name].startBlock
 		rsrc.frequency = self.resources[name].frequency
 	}
@@ -386,7 +405,7 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource,
 }
 
 // update mutable resource index map with specified content
-func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, nextblock uint64, version uint64, indexname *string) (*resource, error) {
+func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, indexname *string) (*resource, error) {
 
 	// rsrc update data chunks are total hacks
 	// and have no size prefix :D
@@ -396,18 +415,36 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, n
 	}
 
 	// update our rsrcs entry map
-	rsrc.lastBlock = nextblock
+	period, version, _, data, err := parseUpdate(chunk.SData[signatureLength:])
+	rsrc.lastPeriod = period
 	rsrc.version = version
-	rsrc.data = make([]byte, len(chunk.SData)-signatureLength)
 	rsrc.updated = time.Now()
-	copy(rsrc.data, chunk.SData[signatureLength:])
-	log.Debug("Resource synced", "name", rsrc.name, "key", chunk.Key, "block", nextblock, "version", version)
-	self.resourceLock.Lock()
-	self.resources[*indexname] = rsrc
-	self.resourceLock.Unlock()
+	rsrc.data = make([]byte, len(data))
+	copy(rsrc.data, data)
+	log.Debug("Resource synced", "name", rsrc.name, "key", chunk.Key, "period", rsrc.lastPeriod, "version", rsrc.version)
+	self.setResource(*indexname, rsrc)
 	return rsrc, nil
 }
 
+func parseUpdate(blob []byte) (period uint32, version uint32, ensname []byte, data []byte, err error) {
+	headerlength := binary.LittleEndian.Uint16(blob[:2])
+	if int(headerlength+2) > len(blob) {
+		return 0, 0, nil, nil, fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(blob))
+	}
+	cursor := 2
+	period = binary.LittleEndian.Uint32(blob[cursor : cursor+4])
+	cursor += 4
+	version = binary.LittleEndian.Uint32(blob[cursor : cursor+4])
+	cursor += 4
+	namelength := int(headerlength) - cursor + 2
+	ensname = make([]byte, namelength)
+	copy(ensname, blob[cursor:])
+	cursor += namelength
+	data = make([]byte, len(blob)-cursor)
+	copy(data, blob[cursor:])
+	return
+}
+
 // Adds an actual data update
 //
 // Uses the data currently loaded in the resources map entry.
@@ -416,6 +453,13 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, n
 // A resource update cannot span chunks, and thus has max length 4096
 func (self *ResourceHandler) Update(name string, data []byte) (Key, error) {
 
+	ok, err := self.validator.isOwner(name)
+	if err != nil {
+		return nil, err
+	} else if !ok {
+		return nil, fmt.Errorf("Not owner of '%s'", name)
+	}
+
 	// can be only one chunk long minus 65 byte signature
 	if int64(len(data)) > self.maxChunkData {
 		return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), 4096-signatureLength)
@@ -436,28 +480,48 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) {
 	if err != nil {
 		return nil, err
 	}
-	nextblock := getNextBlock(resource.startBlock, currentblock, resource.frequency)
+	nextperiod := getNextPeriod(resource.startBlock, currentblock, resource.frequency)
 
 	// if we already have an update for this block then increment version
-	var version uint64
-	if nextblock == resource.lastBlock {
+	var version uint32
+	if self.hasUpdate(name, nextperiod) {
 		version = resource.version
 	}
 	version++
 
+	// prepend version and period to allow reverse lookups
+	// data header length does NOT include the header length prefix bytes themselves
+	headerlength := uint16(len(resource.nameHash) + 4 + 4)
+	fulldata := make([]byte, int(headerlength)+2+len(data))
+
+	cursor := 0
+	binary.LittleEndian.PutUint16(fulldata, headerlength)
+	cursor += 2
+
+	binary.LittleEndian.PutUint32(fulldata[cursor:], nextperiod)
+	cursor += 4
+
+	binary.LittleEndian.PutUint32(fulldata[cursor:], version)
+	cursor += 4
+
+	copy(fulldata[cursor:], resource.nameHash[:])
+	cursor += len(resource.nameHash)
+
+	copy(fulldata[cursor:], data)
+
 	// create the update chunk and send it
-	key := self.resourceHash(resource.ensName, nextblock, version)
+	key := self.resourceHash(resource.nameHash, nextperiod, version)
 	chunk := NewChunk(key, nil)
-	chunk.SData, err = self.signContent(data)
+	chunk.SData, err = self.signContent(fulldata)
 	if err != nil {
 		return nil, err
 	}
-	chunk.Size = int64(len(data))
+	chunk.Size = int64(len(fulldata))
 	self.Put(chunk)
-	log.Trace("resource update", "name", resource.name, "key", key, "currentblock", currentblock, "lastBlock", nextblock, "version", version)
+	log.Trace("resource update", "name", resource.name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData)
 
 	// update our resources map entry and return the new key
-	resource.lastBlock = nextblock
+	resource.lastPeriod = nextperiod
 	resource.version = version
 	resource.data = make([]byte, len(data))
 	copy(resource.data, data)
@@ -473,7 +537,7 @@ func (self *ResourceHandler) Close() {
 func (self *ResourceHandler) getBlock() (uint64, error) {
 	// get the block height and convert to uint64
 	var currentblock string
-	err := self.ethapi.Call(&currentblock, "eth_blockNumber")
+	err := self.rpcClient.Call(&currentblock, "eth_blockNumber")
 	if err != nil {
 		return 0, err
 	}
@@ -483,20 +547,37 @@ func (self *ResourceHandler) getBlock() (uint64, error) {
 	return strconv.ParseUint(currentblock, 10, 64)
 }
 
-func (self *ResourceHandler) resourceHash(namehash common.Hash, blockheight uint64, version uint64) Key {
-	// format is: hash(namehash|blockheight|version)
+func (self *ResourceHandler) BlockToPeriod(name string, blocknumber uint64) uint32 {
+	return getNextPeriod(self.resources[name].startBlock, blocknumber, self.resources[name].frequency)
+}
+
+func (self *ResourceHandler) PeriodToBlock(name string, period uint32) uint64 {
+	return self.resources[name].startBlock + (uint64(period) * self.resources[name].frequency)
+}
+
+func (self *ResourceHandler) getResource(name string) *resource {
+	self.resourceLock.RLock()
+	defer self.resourceLock.RUnlock()
+	rsrc := self.resources[name]
+	return rsrc
+}
+
+func (self *ResourceHandler) setResource(name string, rsrc *resource) {
+	self.resourceLock.Lock()
+	defer self.resourceLock.Unlock()
+	self.resources[name] = rsrc
+}
+
+func (self *ResourceHandler) resourceHash(namehash common.Hash, period uint32, version uint32) Key {
+	// format is: hash(namehash|period|version)
 	self.hashLock.Lock()
 	defer self.hashLock.Unlock()
 	self.hasher.Reset()
 	self.hasher.Write(namehash[:])
-	b := make([]byte, 8)
-	c := binary.PutUvarint(b, blockheight)
+	b := make([]byte, 4)
+	binary.LittleEndian.PutUint32(b, period)
 	self.hasher.Write(b)
-	// PutUvarint only overwrites first c bytes
-	for i := 0; i < c; i++ {
-		b[i] = 0
-	}
-	c = binary.PutUvarint(b, version)
+	binary.LittleEndian.PutUint32(b, version)
 	self.hasher.Write(b)
 	return self.hasher.Sum(nil)
 }
@@ -543,8 +624,57 @@ func (self *ResourceHandler) verifyContent(chunkdata []byte) error {
 	return nil
 }
 
-func getNextBlock(start uint64, current uint64, frequency uint64) uint64 {
-	blockdiff := current - start
-	periods := (blockdiff / frequency) + 1
-	return start + (frequency * periods)
+func (self *ResourceHandler) hasUpdate(name string, period uint32) bool {
+	if self.resources[name].lastPeriod == period {
+		return true
+	}
+	return false
+}
+
+type resourceChunkStore struct {
+	localStore ChunkStore
+	netStore   ChunkStore
+}
+
+func newResourceChunkStore(path string, hasher SwarmHasher, localStore *LocalStore, request func(*Chunk) error) *resourceChunkStore {
+	return &resourceChunkStore{
+		localStore: localStore,
+		netStore:   NewDpaChunkStore(localStore, request),
+	}
+}
+
+func (r *resourceChunkStore) Get(key Key) (*Chunk, error) {
+	chunk, err := r.netStore.Get(key)
+	if err != nil {
+		return nil, err
+	}
+	// if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing.
+	// sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue
+	// that caused the failure or that the chunk doesn't exist.
+	if chunk.ReqC == nil {
+		return chunk, nil
+	}
+	t := time.NewTimer(time.Second * 1)
+	select {
+	case <-t.C:
+		return nil, fmt.Errorf("timeout")
+	case <-chunk.C:
+		log.Trace("Received resource update chunk")
+	}
+	return chunk, nil
+}
+
+func (r *resourceChunkStore) Put(chunk *Chunk) {
+	r.netStore.Put(chunk)
+}
+
+func (r *resourceChunkStore) Close() {
+	r.netStore.Close()
+	r.localStore.Close()
+}
+
+func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 {
+	blockdiff := current - start
+	period := blockdiff / frequency
+	return uint32(period + 1)
 }

swarm/storage/resource_ens.go

@@ -0,0 +1,54 @@
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/contracts/ens"
+)
+
+// ENS validation of mutable resource owners
+type ENSValidator struct {
+	owner common.Address
+	api   *ens.ENS
+}
+
+func NewENSValidator(owneraddress common.Address, contractaddress common.Address, backend bind.ContractBackend, transactOpts *bind.TransactOpts) (*ENSValidator, error) {
+	var err error
+	validator := &ENSValidator{}
+	validator.api, err = ens.NewENS(transactOpts, contractaddress, backend)
+	if err != nil {
+		return nil, err
+	}
+	validator.owner = owneraddress
+	return validator, nil
+}
+
+func (self *ENSValidator) isOwner(name string) (bool, error) {
+	owneraddr, err := self.api.Owner(self.nameHash(name))
+	if err != nil {
+		return false, err
+	}
+	return owneraddr == self.owner, nil
+}
+
+func (self *ENSValidator) nameHash(name string) common.Hash {
+	return ens.EnsNode(name)
+}
+
+// Default fallthrough validation of mutable resource ownership
+type GenericValidator struct {
+	hashFunc func(string) common.Hash
+}
+
+func NewGenericValidator(hashFunc func(string) common.Hash) *GenericValidator {
+	return &GenericValidator{
+		hashFunc: hashFunc,
+	}
+}
+func (self *GenericValidator) isOwner(name string) (bool, error) {
+	return true, nil
+}
+
+func (self *GenericValidator) nameHash(name string) common.Hash {
+	return self.hashFunc(name)
+}

swarm/storage/resource_test.go

@@ -7,47 +7,85 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io/ioutil"
+	"math/big"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	"golang.org/x/net/idna"
 
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/contracts/ens"
+	"github.com/ethereum/go-ethereum/contracts/ens/contract"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
 var (
-	blockCount = uint64(4200)
-	cleanF     func()
+	hasher            = MakeHashFunc("SHA3")()
+	zeroAddr          = common.Address{}
+	startBlock        = uint64(4200)
+	resourceFrequency = uint64(42)
+	cleanF            func()
+	domainName        = "føø.bar"
 )
 
+func init() {
+	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
+}
+
+// simulated backend does not have the blocknumber call
+// so we use this wrapper to fake returning the block count
+type fakeBackend struct {
+	*backends.SimulatedBackend
+	blocknumber uint64
+}
+
+func (f *fakeBackend) Commit() {
+	if f.SimulatedBackend != nil {
+		f.SimulatedBackend.Commit()
+	}
+	f.blocknumber++
+}
+
 type FakeRPC struct {
-	blockcount *uint64
+	backend *fakeBackend
 }
 
 func (r *FakeRPC) BlockNumber() (string, error) {
-	return strconv.FormatUint(*r.blockcount, 10), nil
+	return strconv.FormatUint(r.backend.blocknumber, 10), nil
 }
 
-func TestResourceValidContent(t *testing.T) {
+// check that signature address matches update signer address
+func TestResourceSignature(t *testing.T) {
 
-	rh, privkey, _, err, teardownTest := setupTest()
+	// privkey for signing updates
+	privkey, err := crypto.GenerateKey()
+	if err != nil {
+		return
+	}
+
+	// set up rpc and create resourcehandler
+	rh, _, err, teardownTest := setupTest(privkey, nil, nil)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// create a new resource
-	validname, err := idna.ToASCII("føø.bar")
+	validname, err := idna.ToASCII(domainName)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// generate a hash for block 4200 version 1
-	key := rh.resourceHash(ens.EnsNode(validname), 4200, 1)
+	key := rh.resourceHash(ens.EnsNode(validname), 1, 1)
 	chunk := NewChunk(key, nil)
 
 	// generate some bogus data for the chunk and sign it
@@ -56,9 +94,9 @@ func TestResourceValidContent(t *testing.T) {
 	if err != nil {
 		teardownTest(t, err)
 	}
-	rh.hasher.Reset()
-	rh.hasher.Write(data)
-	datahash := rh.hasher.Sum(nil)
+	hasher.Reset()
+	hasher.Write(data)
+	datahash := hasher.Sum(nil)
 	sig, err := crypto.Sign(datahash, privkey)
 	if err != nil {
 		teardownTest(t, err)
@@ -83,28 +121,97 @@ func TestResourceValidContent(t *testing.T) {
 	teardownTest(t, nil)
 }
 
-func TestResourceHandler(t *testing.T) {
+// determine resource update metadata from chunk data
+func TestResourceReverseLookup(t *testing.T) {
 
-	rh, privkey, datadir, err, teardownTest := setupTest()
+	// privkey for signing updates
+	privkey, err := crypto.GenerateKey()
+	if err != nil {
+		return
+	}
+
+	// make fake backend, set up rpc and create resourcehandler
+	backend := &fakeBackend{
+		blocknumber: startBlock,
+	}
+	rh, _, err, teardownTest := setupTest(privkey, backend, nil)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// create a new resource
-	resourcename := "føø.bar"
-	resourcevalidname, err := idna.ToASCII(resourcename)
+	rsrc, err := rh.NewResource(domainName, resourceFrequency)
 	if err != nil {
 		teardownTest(t, err)
 	}
-	resourcefrequency := uint64(42)
-	_, err = rh.NewResource(resourcename, resourcefrequency)
+
+	// update data
+	fwdBlocks(int(resourceFrequency+1), backend)
+	data := []byte("foo")
+	resourcekey, err := rh.Update(domainName, data)
+	if err != nil {
+		teardownTest(t, err)
+	}
+	chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(resourcekey))
+	if err != nil {
+		teardownTest(t, err)
+	}
+
+	// check if data after header length offset is as expected
+	headerlength := binary.LittleEndian.Uint16(chunk.SData[signatureLength : signatureLength+2])
+	if !bytes.Equal(chunk.SData[signatureLength+headerlength+2:], data) {
+		teardownTest(t, fmt.Errorf("Expected chunk data with header length %d (pos %d) to match %x, but was %x", headerlength, signatureLength+headerlength+2, data, chunk.SData[signatureLength+headerlength+2:]))
+	}
+
+	// get name, period, version from chunk and check
+	revperiod, revversion, revname, revdata, err := parseUpdate(chunk.SData[signatureLength:])
+
+	if !bytes.Equal(revname, rsrc.nameHash.Bytes()) {
+		teardownTest(t, fmt.Errorf("Expected retrieved name from chunk data to be '%x', was '%x'", rsrc.nameHash.Bytes(), revname))
+	}
+	if !bytes.Equal(revdata, data) {
+		teardownTest(t, fmt.Errorf("Expected retrieved data from chunk data to be '%x', was '%x'", data, revdata))
+	}
+
+	if revperiod != 2 {
+		teardownTest(t, fmt.Errorf("Expected retrieved period from chunk data to be 1, was %d", revperiod))
+	}
+	if revversion != 1 {
+		teardownTest(t, fmt.Errorf("Expected retrieved version from chunk data to be 1, was %d", revversion))
+	}
+}
+
+// make updates and retrieve them based on periods and versions
+func TestResourceHandler(t *testing.T) {
+
+	// privkey for signing updates
+	privkey, err := crypto.GenerateKey()
+	if err != nil {
+		return
+	}
+
+	// make fake backend, set up rpc and create resourcehandler
+	backend := &fakeBackend{
+		blocknumber: startBlock,
+	}
+	rh, datadir, err, teardownTest := setupTest(privkey, backend, nil)
+	if err != nil {
+		teardownTest(t, err)
+	}
+
+	// create a new resource
+	resourcevalidname, err := idna.ToASCII(domainName)
+	if err != nil {
+		teardownTest(t, err)
+	}
+	_, err = rh.NewResource(domainName, resourceFrequency)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// check that the new resource is stored correctly
-	namehash := ens.EnsNode(resourcevalidname)
-	chunk, err := rh.ChunkStore.Get(Key(namehash[:]))
+	namehash := rh.validator.nameHash(resourcevalidname)
+	chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:]))
 	if err != nil {
 		teardownTest(t, err)
 	} else if len(chunk.SData) < 16 {
@@ -112,38 +219,38 @@ func TestResourceHandler(t *testing.T) {
 	}
 	startblocknumber := binary.LittleEndian.Uint64(chunk.SData[8:16])
 	chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[16:])
-	if startblocknumber != blockCount {
-		teardownTest(t, fmt.Errorf("stored block number %d does not match provided block number %d", startblocknumber, blockCount))
+	if startblocknumber != backend.blocknumber {
+		teardownTest(t, fmt.Errorf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber))
 	}
-	if chunkfrequency != resourcefrequency {
-		teardownTest(t, fmt.Errorf("stored frequency %d does not match provided frequency %d", chunkfrequency, resourcefrequency))
+	if chunkfrequency != resourceFrequency {
+		teardownTest(t, fmt.Errorf("stored frequency %d does not match provided frequency %d", chunkfrequency, resourceFrequency))
 	}
 
 	// update halfway to first period
 	resourcekey := make(map[string]Key)
-	blockCount = startblocknumber + (resourcefrequency / 2)
-	resourcekey["blinky"], err = rh.Update(resourcename, []byte("blinky"))
+	fwdBlocks(int(resourceFrequency/2), backend)
+	resourcekey["blinky"], err = rh.Update(domainName, []byte("blinky"))
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// update on first period
-	blockCount = startblocknumber + resourcefrequency
-	resourcekey["pinky"], err = rh.Update(resourcename, []byte("pinky"))
+	fwdBlocks(int(resourceFrequency/2), backend)
+	resourcekey["pinky"], err = rh.Update(domainName, []byte("pinky"))
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// update on second period
-	blockCount = startblocknumber + (resourcefrequency * 2)
-	resourcekey["inky"], err = rh.Update(resourcename, []byte("inky"))
+	fwdBlocks(int(resourceFrequency), backend)
+	resourcekey["inky"], err = rh.Update(domainName, []byte("inky"))
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// update just after second period
-	blockCount = startblocknumber + (resourcefrequency * 2) + 1
-	resourcekey["clyde"], err = rh.Update(resourcename, []byte("clyde"))
+	fwdBlocks(1, backend)
+	resourcekey["clyde"], err = rh.Update(domainName, []byte("clyde"))
 	if err != nil {
 		teardownTest(t, err)
 	}
@@ -151,75 +258,149 @@ func TestResourceHandler(t *testing.T) {
 	rh.Close()
 
 	// check we can retrieve the updates after close
-	// it will match on second iteration startblocknumber + (resourcefrequency * 3)
-	blockCount = startblocknumber + (resourcefrequency * 4)
+	// it will match on second iteration startblocknumber + (resourceFrequency * 3)
+	fwdBlocks(int(resourceFrequency*2)-1, backend)
 
-	rh2, err := newTestResourceHandler(datadir, privkey, rh.ethapi)
+	rh2, err := newTestResourceHandler(datadir, privkey, rh.rpcClient, nil)
 	if err != nil {
 		teardownTest(t, err)
 	}
-	_, err = rh2.LookupLatest(resourcename, true)
+	_, err = rh2.LookupLatest(domainName, true)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// last update should be "clyde", version two, blockheight startblocknumber + (resourcefrequency * 3)
-	if !bytes.Equal(rh2.resources[resourcename].data, []byte("clyde")) {
-		teardownTest(t, fmt.Errorf("resource data was %v, expected %v", rh2.resources[resourcename].data, []byte("clyde")))
+	if !bytes.Equal(rh2.resources[domainName].data, []byte("clyde")) {
+		teardownTest(t, fmt.Errorf("resource data was %v, expected %v", rh2.resources[domainName].data, []byte("clyde")))
 	}
-	if rh2.resources[resourcename].version != 2 {
-		teardownTest(t, fmt.Errorf("resource version was %d, expected 2", rh2.resources[resourcename].version))
+	if rh2.resources[domainName].version != 2 {
+		teardownTest(t, fmt.Errorf("resource version was %d, expected 2", rh2.resources[domainName].version))
 	}
-	if rh2.resources[resourcename].lastBlock != startblocknumber+(resourcefrequency*3) {
-		teardownTest(t, fmt.Errorf("resource blockheight was %d, expected %d", rh2.resources[resourcename].lastBlock, startblocknumber+(resourcefrequency*3)))
+	if rh2.resources[domainName].lastPeriod != 3 {
+		teardownTest(t, fmt.Errorf("resource period was %d, expected 3", rh2.resources[domainName].lastPeriod))
 	}
 
-	rsrc, err := NewResource(resourcename, startblocknumber, resourcefrequency)
+	rsrc, err := NewResource(domainName, startblocknumber, resourceFrequency, rh2.validator.nameHash)
 	if err != nil {
 		teardownTest(t, err)
 	}
-	err = rh2.SetResource(rsrc, true)
+	err = rh2.SetExternalResource(rsrc, true)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// latest block, latest version
-	resource, err := rh2.LookupLatest(resourcename, false) // if key is specified, refresh is implicit
+	resource, err := rh2.LookupLatest(domainName, false) // if key is specified, refresh is implicit
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// check data
 	if !bytes.Equal(resource.data, []byte("clyde")) {
-		teardownTest(t, fmt.Errorf("resource data (latest) was %v, expected %v", rh2.resources[resourcename].data, []byte("clyde")))
+		teardownTest(t, fmt.Errorf("resource data (latest) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde")))
 	}
 
 	// specific block, latest version
-	resource, err = rh2.LookupHistorical(resourcename, startblocknumber+(resourcefrequency*3), true)
+	resource, err = rh2.LookupHistorical(domainName, 3, true)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// check data
 	if !bytes.Equal(resource.data, []byte("clyde")) {
-		teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[resourcename].data, []byte("clyde")))
+		teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde")))
 	}
 
 	// specific block, specific version
-	resource, err = rh2.LookupVersion(resourcename, startblocknumber+(resourcefrequency*3), 1, true)
+	resource, err = rh2.LookupVersion(domainName, 3, 1, true)
 	if err != nil {
 		teardownTest(t, err)
 	}
 
 	// check data
 	if !bytes.Equal(resource.data, []byte("inky")) {
-		teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[resourcename].data, []byte("inky")))
+		teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky")))
 	}
 	teardownTest(t, nil)
 
 }
 
-func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string, err error, teardown func(*testing.T, error)) {
+// create ENS enabled resource update, with and without valid owner
+func TestResourceENSOwner(t *testing.T) {
+
+	// privkey for signing updates
+	privkey, err := crypto.GenerateKey()
+	if err != nil {
+		return
+	}
+
+	// privkey for checking wrong owner
+	privkeytwo, err := crypto.GenerateKey()
+	if err != nil {
+		return
+	}
+
+	// ens address and transact options
+	addr := crypto.PubkeyToAddress(privkey.PublicKey)
+	transactOpts := bind.NewKeyedTransactor(privkey)
+
+	// set up ENS sim
+	domainparts := strings.Split(domainName, ".")
+	contractAddr, contractbackend, err := setupENS(addr, transactOpts, domainparts[0], domainparts[1])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	validator, err := NewENSValidator(addr, contractAddr, contractbackend, transactOpts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// set up rpc and create resourcehandler with ENS sim backend
+	rh, _, err, teardownTest := setupTest(privkey, contractbackend, validator)
+	if err != nil {
+		teardownTest(t, err)
+	}
+
+	// create new resource when we are owner = ok
+	_, err = rh.NewResource(domainName, 42)
+	if err != nil {
+		teardownTest(t, fmt.Errorf("Create resource fail: %v", err))
+	}
+
+	// update resource when we are owner = ok
+	_, err = rh.Update(domainName, []byte("foo"))
+	if err != nil {
+		teardownTest(t, fmt.Errorf("Update resource fail: %v", err))
+	}
+
+	// create new resource when we are NOT owner = !ok
+	addrtwo := crypto.PubkeyToAddress(privkeytwo.PublicKey)
+	validator.owner = addrtwo
+
+	_, err = rh.NewResource(domainName, 42)
+	if err == nil {
+		teardownTest(t, fmt.Errorf("Expected resource create fail due to owner mismatch"))
+	}
+	// update resource when we are owner = ok
+	_, err = rh.Update(domainName, []byte("foo"))
+	if err == nil {
+		teardownTest(t, fmt.Errorf("Expected resource update fail due to owner mismatch"))
+	}
+
+	teardownTest(t, nil)
+}
+
+// fast-forward blockheight
+func fwdBlocks(count int, backend *fakeBackend) {
+	for i := 0; i < count; i++ {
+		backend.Commit()
+	}
+}
+
+// create rpc and resourcehandler
+func setupTest(privkey *ecdsa.PrivateKey, contractbackend bind.ContractBackend, validator ResourceValidator) (rh *ResourceHandler, datadir string, err error, teardown func(*testing.T, error)) {
 
 	var fsClean func()
 	var rpcClean func()
@@ -232,12 +413,6 @@ func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string
 		}
 	}
 
-	// privkey for signing updates
-	privkey, err = crypto.GenerateKey()
-	if err != nil {
-		return
-	}
-
 	// temp datadir
 	datadir, err = ioutil.TempDir("", "rh")
 	if err != nil {
@@ -255,8 +430,12 @@ func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string
 		return
 	}
 	rpcserver := rpc.NewServer()
+	var fake *fakeBackend
+	if contractbackend != nil {
+		fake = contractbackend.(*fakeBackend)
+	}
 	rpcserver.RegisterName("eth", &FakeRPC{
-		blockcount: &blockCount,
+		backend: fake,
 	})
 	go func() {
 		rpcserver.ServeListener(ipcl)
@@ -271,7 +450,8 @@ func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string
 		return
 	}
 
-	rh, err = newTestResourceHandler(datadir, privkey, rpcclient)
+	// choose if with ens or not
+	rh, err = newTestResourceHandler(datadir, privkey, rpcclient, validator)
 
 	teardown = func(t *testing.T, err error) {
 		cleanF()
@@ -283,7 +463,7 @@ func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string
 	return
 }
 
-func newTestResourceHandler(datadir string, privkey *ecdsa.PrivateKey, rpcclient *rpc.Client) (*ResourceHandler, error) {
+func newTestResourceHandler(datadir string, privkey *ecdsa.PrivateKey, rpcclient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) {
 	path := filepath.Join(datadir, "resource")
 	basekey := make([]byte, 32)
 	hasher := MakeHashFunc("SHA3")
@@ -295,6 +475,48 @@ func newTestResourceHandler(datadir string, privkey *ecdsa.PrivateKey, rpcclient
 		memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
 		DbStore:  dbStore,
 	}
-
-	return NewResourceHandler(privkey, hasher, localStore, rpcclient)
+	return NewResourceHandler(privkey, hasher, localStore, rpcclient, validator)
+}
+
+// Set up simulated ENS backend for use with ENSResourceHandler tests
+func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, top string) (common.Address, bind.ContractBackend, error) {
+
+	// create the domain hash values to pass to the ENS contract methods
+	var tophash [32]byte
+	var subhash [32]byte
+
+	hasher.Reset()
+	hasher.Write([]byte(top))
+	copy(tophash[:], hasher.Sum(nil))
+	hasher.Reset()
+	hasher.Write([]byte(sub))
+	copy(subhash[:], hasher.Sum(nil))
+
+	// initialize contract backend and deploy
+	contractBackend := &fakeBackend{
+		SimulatedBackend: backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}),
+	}
+
+	contractAddress, _, ensinstance, err := contract.DeployENS(transactOpts, contractBackend)
+	if err != nil {
+		return zeroAddr, nil, fmt.Errorf("can't deploy: %v", err)
+	}
+
+	// update the registry for the correct owner address
+	if _, err = ensinstance.SetOwner(transactOpts, [32]byte{}, addr); err != nil {
+		return zeroAddr, nil, fmt.Errorf("can't setowner: %v", err)
+	}
+	contractBackend.Commit()
+
+	if _, err = ensinstance.SetSubnodeOwner(transactOpts, [32]byte{}, tophash, addr); err != nil {
+		return zeroAddr, nil, fmt.Errorf("can't register top: %v", err)
+	}
+	contractBackend.Commit()
+
+	if _, err = ensinstance.SetSubnodeOwner(transactOpts, ens.EnsNode(top), subhash, addr); err != nil {
+		return zeroAddr, nil, fmt.Errorf("can't register top: %v", err)
+	}
+	contractBackend.Commit()
+
+	return contractAddress, contractBackend, nil
 }

swarm/swarm.go

@@ -41,6 +41,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 )
 
 // the swarm stack
@@ -80,7 +81,9 @@ func (self *Swarm) API() *SwarmAPI {
 
 // creates a new swarm service instance
 // implements node.Service
-func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool) (self *Swarm, err error) {
+// If mockStore is not nil, it will be used as the storage for chunk data.
+// MockStore should be used only for testing.
+func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) {
 	if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) {
 		return nil, fmt.Errorf("empty public key")
 	}
@@ -98,7 +101,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
 	log.Debug(fmt.Sprintf("Setting up Swarm service components"))
 
 	hash := storage.MakeHashFunc(config.ChunkerParams.Hash)
-	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.Hex2Bytes(config.BzzKey))
+	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.Hex2Bytes(config.BzzKey), mockStore)
 	if err != nil {
 		return
 	}

vendor/golang.org/x/net/idna/idna.go

@@ -0,0 +1,732 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package idna implements IDNA2008 using the compatibility processing
+// defined by UTS (Unicode Technical Standard) #46, which defines a standard to
+// deal with the transition from IDNA2003.
+//
+// IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC
+// 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894.
+// UTS #46 is defined in http://www.unicode.org/reports/tr46.
+// See http://unicode.org/cldr/utility/idna.jsp for a visualization of the
+// differences between these two standards.
+package idna // import "golang.org/x/net/idna"
+
+import (
+	"fmt"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/text/secure/bidirule"
+	"golang.org/x/text/unicode/bidi"
+	"golang.org/x/text/unicode/norm"
+)
+
+// NOTE: Unlike common practice in Go APIs, the functions will return a
+// sanitized domain name in case of errors. Browsers sometimes use a partially
+// evaluated string as lookup.
+// TODO: the current error handling is, in my opinion, the least opinionated.
+// Other strategies are also viable, though:
+// Option 1) Return an empty string in case of error, but allow the user to
+//    specify explicitly which errors to ignore.
+// Option 2) Return the partially evaluated string if it is itself a valid
+//    string, otherwise return the empty string in case of error.
+// Option 3) Option 1 and 2.
+// Option 4) Always return an empty string for now and implement Option 1 as
+//    needed, and document that the return string may not be empty in case of
+//    error in the future.
+// I think Option 1 is best, but it is quite opinionated.
+
+// ToASCII is a wrapper for Punycode.ToASCII.
+func ToASCII(s string) (string, error) {
+	return Punycode.process(s, true)
+}
+
+// ToUnicode is a wrapper for Punycode.ToUnicode.
+func ToUnicode(s string) (string, error) {
+	return Punycode.process(s, false)
+}
+
+// An Option configures a Profile at creation time.
+type Option func(*options)
+
+// Transitional sets a Profile to use the Transitional mapping as defined in UTS
+// #46. This will cause, for example, "ß" to be mapped to "ss". Using the
+// transitional mapping provides a compromise between IDNA2003 and IDNA2008
+// compatibility. It is used by most browsers when resolving domain names. This
+// option is only meaningful if combined with MapForLookup.
+func Transitional(transitional bool) Option {
+	return func(o *options) { o.transitional = true }
+}
+
+// VerifyDNSLength sets whether a Profile should fail if any of the IDN parts
+// are longer than allowed by the RFC.
+func VerifyDNSLength(verify bool) Option {
+	return func(o *options) { o.verifyDNSLength = verify }
+}
+
+// RemoveLeadingDots removes leading label separators. Leading runes that map to
+// dots, such as U+3002 IDEOGRAPHIC FULL STOP, are removed as well.
+//
+// This is the behavior suggested by the UTS #46 and is adopted by some
+// browsers.
+func RemoveLeadingDots(remove bool) Option {
+	return func(o *options) { o.removeLeadingDots = remove }
+}
+
+// ValidateLabels sets whether to check the mandatory label validation criteria
+// as defined in Section 5.4 of RFC 5891. This includes testing for correct use
+// of hyphens ('-'), normalization, validity of runes, and the context rules.
+func ValidateLabels(enable bool) Option {
+	return func(o *options) {
+		// Don't override existing mappings, but set one that at least checks
+		// normalization if it is not set.
+		if o.mapping == nil && enable {
+			o.mapping = normalize
+		}
+		o.trie = trie
+		o.validateLabels = enable
+		o.fromPuny = validateFromPunycode
+	}
+}
+
+// StrictDomainName limits the set of permissible ASCII characters to those
+// allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the
+// hyphen). This is set by default for MapForLookup and ValidateForRegistration.
+//
+// This option is useful, for instance, for browsers that allow characters
+// outside this range, for example a '_' (U+005F LOW LINE). See
+// http://www.rfc-editor.org/std/std3.txt for more details This option
+// corresponds to the UseSTD3ASCIIRules option in UTS #46.
+func StrictDomainName(use bool) Option {
+	return func(o *options) {
+		o.trie = trie
+		o.useSTD3Rules = use
+		o.fromPuny = validateFromPunycode
+	}
+}
+
+// NOTE: the following options pull in tables. The tables should not be linked
+// in as long as the options are not used.
+
+// BidiRule enables the Bidi rule as defined in RFC 5893. Any application
+// that relies on proper validation of labels should include this rule.
+func BidiRule() Option {
+	return func(o *options) { o.bidirule = bidirule.ValidString }
+}
+
+// ValidateForRegistration sets validation options to verify that a given IDN is
+// properly formatted for registration as defined by Section 4 of RFC 5891.
+func ValidateForRegistration() Option {
+	return func(o *options) {
+		o.mapping = validateRegistration
+		StrictDomainName(true)(o)
+		ValidateLabels(true)(o)
+		VerifyDNSLength(true)(o)
+		BidiRule()(o)
+	}
+}
+
+// MapForLookup sets validation and mapping options such that a given IDN is
+// transformed for domain name lookup according to the requirements set out in
+// Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894,
+// RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option
+// to add this check.
+//
+// The mappings include normalization and mapping case, width and other
+// compatibility mappings.
+func MapForLookup() Option {
+	return func(o *options) {
+		o.mapping = validateAndMap
+		StrictDomainName(true)(o)
+		ValidateLabels(true)(o)
+	}
+}
+
+type options struct {
+	transitional      bool
+	useSTD3Rules      bool
+	validateLabels    bool
+	verifyDNSLength   bool
+	removeLeadingDots bool
+
+	trie *idnaTrie
+
+	// fromPuny calls validation rules when converting A-labels to U-labels.
+	fromPuny func(p *Profile, s string) error
+
+	// mapping implements a validation and mapping step as defined in RFC 5895
+	// or UTS 46, tailored to, for example, domain registration or lookup.
+	mapping func(p *Profile, s string) (mapped string, isBidi bool, err error)
+
+	// bidirule, if specified, checks whether s conforms to the Bidi Rule
+	// defined in RFC 5893.
+	bidirule func(s string) bool
+}
+
+// A Profile defines the configuration of an IDNA mapper.
+type Profile struct {
+	options
+}
+
+func apply(o *options, opts []Option) {
+	for _, f := range opts {
+		f(o)
+	}
+}
+
+// New creates a new Profile.
+//
+// With no options, the returned Profile is the most permissive and equals the
+// Punycode Profile. Options can be passed to further restrict the Profile. The
+// MapForLookup and ValidateForRegistration options set a collection of options,
+// for lookup and registration purposes respectively, which can be tailored by
+// adding more fine-grained options, where later options override earlier
+// options.
+func New(o ...Option) *Profile {
+	p := &Profile{}
+	apply(&p.options, o)
+	return p
+}
+
+// ToASCII converts a domain or domain label to its ASCII form. For example,
+// ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and
+// ToASCII("golang") is "golang". If an error is encountered it will return
+// an error and a (partially) processed result.
+func (p *Profile) ToASCII(s string) (string, error) {
+	return p.process(s, true)
+}
+
+// ToUnicode converts a domain or domain label to its Unicode form. For example,
+// ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and
+// ToUnicode("golang") is "golang". If an error is encountered it will return
+// an error and a (partially) processed result.
+func (p *Profile) ToUnicode(s string) (string, error) {
+	pp := *p
+	pp.transitional = false
+	return pp.process(s, false)
+}
+
+// String reports a string with a description of the profile for debugging
+// purposes. The string format may change with different versions.
+func (p *Profile) String() string {
+	s := ""
+	if p.transitional {
+		s = "Transitional"
+	} else {
+		s = "NonTransitional"
+	}
+	if p.useSTD3Rules {
+		s += ":UseSTD3Rules"
+	}
+	if p.validateLabels {
+		s += ":ValidateLabels"
+	}
+	if p.verifyDNSLength {
+		s += ":VerifyDNSLength"
+	}
+	return s
+}
+
+var (
+	// Punycode is a Profile that does raw punycode processing with a minimum
+	// of validation.
+	Punycode *Profile = punycode
+
+	// Lookup is the recommended profile for looking up domain names, according
+	// to Section 5 of RFC 5891. The exact configuration of this profile may
+	// change over time.
+	Lookup *Profile = lookup
+
+	// Display is the recommended profile for displaying domain names.
+	// The configuration of this profile may change over time.
+	Display *Profile = display
+
+	// Registration is the recommended profile for checking whether a given
+	// IDN is valid for registration, according to Section 4 of RFC 5891.
+	Registration *Profile = registration
+
+	punycode = &Profile{}
+	lookup   = &Profile{options{
+		transitional:   true,
+		useSTD3Rules:   true,
+		validateLabels: true,
+		trie:           trie,
+		fromPuny:       validateFromPunycode,
+		mapping:        validateAndMap,
+		bidirule:       bidirule.ValidString,
+	}}
+	display = &Profile{options{
+		useSTD3Rules:   true,
+		validateLabels: true,
+		trie:           trie,
+		fromPuny:       validateFromPunycode,
+		mapping:        validateAndMap,
+		bidirule:       bidirule.ValidString,
+	}}
+	registration = &Profile{options{
+		useSTD3Rules:    true,
+		validateLabels:  true,
+		verifyDNSLength: true,
+		trie:            trie,
+		fromPuny:        validateFromPunycode,
+		mapping:         validateRegistration,
+		bidirule:        bidirule.ValidString,
+	}}
+
+	// TODO: profiles
+	// Register: recommended for approving domain names: don't do any mappings
+	// but rather reject on invalid input. Bundle or block deviation characters.
+)
+
+type labelError struct{ label, code_ string }
+
+func (e labelError) code() string { return e.code_ }
+func (e labelError) Error() string {
+	return fmt.Sprintf("idna: invalid label %q", e.label)
+}
+
+type runeError rune
+
+func (e runeError) code() string { return "P1" }
+func (e runeError) Error() string {
+	return fmt.Sprintf("idna: disallowed rune %U", e)
+}
+
+// process implements the algorithm described in section 4 of UTS #46,
+// see http://www.unicode.org/reports/tr46.
+func (p *Profile) process(s string, toASCII bool) (string, error) {
+	var err error
+	var isBidi bool
+	if p.mapping != nil {
+		s, isBidi, err = p.mapping(p, s)
+	}
+	// Remove leading empty labels.
+	if p.removeLeadingDots {
+		for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
+		}
+	}
+	// TODO: allow for a quick check of the tables data.
+	// It seems like we should only create this error on ToASCII, but the
+	// UTS 46 conformance tests suggests we should always check this.
+	if err == nil && p.verifyDNSLength && s == "" {
+		err = &labelError{s, "A4"}
+	}
+	labels := labelIter{orig: s}
+	for ; !labels.done(); labels.next() {
+		label := labels.label()
+		if label == "" {
+			// Empty labels are not okay. The label iterator skips the last
+			// label if it is empty.
+			if err == nil && p.verifyDNSLength {
+				err = &labelError{s, "A4"}
+			}
+			continue
+		}
+		if strings.HasPrefix(label, acePrefix) {
+			u, err2 := decode(label[len(acePrefix):])
+			if err2 != nil {
+				if err == nil {
+					err = err2
+				}
+				// Spec says keep the old label.
+				continue
+			}
+			isBidi = isBidi || bidirule.DirectionString(u) != bidi.LeftToRight
+			labels.set(u)
+			if err == nil && p.validateLabels {
+				err = p.fromPuny(p, u)
+			}
+			if err == nil {
+				// This should be called on NonTransitional, according to the
+				// spec, but that currently does not have any effect. Use the
+				// original profile to preserve options.
+				err = p.validateLabel(u)
+			}
+		} else if err == nil {
+			err = p.validateLabel(label)
+		}
+	}
+	if isBidi && p.bidirule != nil && err == nil {
+		for labels.reset(); !labels.done(); labels.next() {
+			if !p.bidirule(labels.label()) {
+				err = &labelError{s, "B"}
+				break
+			}
+		}
+	}
+	if toASCII {
+		for labels.reset(); !labels.done(); labels.next() {
+			label := labels.label()
+			if !ascii(label) {
+				a, err2 := encode(acePrefix, label)
+				if err == nil {
+					err = err2
+				}
+				label = a
+				labels.set(a)
+			}
+			n := len(label)
+			if p.verifyDNSLength && err == nil && (n == 0 || n > 63) {
+				err = &labelError{label, "A4"}
+			}
+		}
+	}
+	s = labels.result()
+	if toASCII && p.verifyDNSLength && err == nil {
+		// Compute the length of the domain name minus the root label and its dot.
+		n := len(s)
+		if n > 0 && s[n-1] == '.' {
+			n--
+		}
+		if len(s) < 1 || n > 253 {
+			err = &labelError{s, "A4"}
+		}
+	}
+	return s, err
+}
+
+func normalize(p *Profile, s string) (mapped string, isBidi bool, err error) {
+	// TODO: consider first doing a quick check to see if any of these checks
+	// need to be done. This will make it slower in the general case, but
+	// faster in the common case.
+	mapped = norm.NFC.String(s)
+	isBidi = bidirule.DirectionString(mapped) == bidi.RightToLeft
+	return mapped, isBidi, nil
+}
+
+func validateRegistration(p *Profile, s string) (idem string, bidi bool, err error) {
+	// TODO: filter need for normalization in loop below.
+	if !norm.NFC.IsNormalString(s) {
+		return s, false, &labelError{s, "V1"}
+	}
+	for i := 0; i < len(s); {
+		v, sz := trie.lookupString(s[i:])
+		if sz == 0 {
+			return s, bidi, runeError(utf8.RuneError)
+		}
+		bidi = bidi || info(v).isBidi(s[i:])
+		// Copy bytes not copied so far.
+		switch p.simplify(info(v).category()) {
+		// TODO: handle the NV8 defined in the Unicode idna data set to allow
+		// for strict conformance to IDNA2008.
+		case valid, deviation:
+		case disallowed, mapped, unknown, ignored:
+			r, _ := utf8.DecodeRuneInString(s[i:])
+			return s, bidi, runeError(r)
+		}
+		i += sz
+	}
+	return s, bidi, nil
+}
+
+func (c info) isBidi(s string) bool {
+	if !c.isMapped() {
+		return c&attributesMask == rtl
+	}
+	// TODO: also store bidi info for mapped data. This is possible, but a bit
+	// cumbersome and not for the common case.
+	p, _ := bidi.LookupString(s)
+	switch p.Class() {
+	case bidi.R, bidi.AL, bidi.AN:
+		return true
+	}
+	return false
+}
+
+func validateAndMap(p *Profile, s string) (vm string, bidi bool, err error) {
+	var (
+		b []byte
+		k int
+	)
+	// combinedInfoBits contains the or-ed bits of all runes. We use this
+	// to derive the mayNeedNorm bit later. This may trigger normalization
+	// overeagerly, but it will not do so in the common case. The end result
+	// is another 10% saving on BenchmarkProfile for the common case.
+	var combinedInfoBits info
+	for i := 0; i < len(s); {
+		v, sz := trie.lookupString(s[i:])
+		if sz == 0 {
+			b = append(b, s[k:i]...)
+			b = append(b, "\ufffd"...)
+			k = len(s)
+			if err == nil {
+				err = runeError(utf8.RuneError)
+			}
+			break
+		}
+		combinedInfoBits |= info(v)
+		bidi = bidi || info(v).isBidi(s[i:])
+		start := i
+		i += sz
+		// Copy bytes not copied so far.
+		switch p.simplify(info(v).category()) {
+		case valid:
+			continue
+		case disallowed:
+			if err == nil {
+				r, _ := utf8.DecodeRuneInString(s[start:])
+				err = runeError(r)
+			}
+			continue
+		case mapped, deviation:
+			b = append(b, s[k:start]...)
+			b = info(v).appendMapping(b, s[start:i])
+		case ignored:
+			b = append(b, s[k:start]...)
+			// drop the rune
+		case unknown:
+			b = append(b, s[k:start]...)
+			b = append(b, "\ufffd"...)
+		}
+		k = i
+	}
+	if k == 0 {
+		// No changes so far.
+		if combinedInfoBits&mayNeedNorm != 0 {
+			s = norm.NFC.String(s)
+		}
+	} else {
+		b = append(b, s[k:]...)
+		if norm.NFC.QuickSpan(b) != len(b) {
+			b = norm.NFC.Bytes(b)
+		}
+		// TODO: the punycode converters require strings as input.
+		s = string(b)
+	}
+	return s, bidi, err
+}
+
+// A labelIter allows iterating over domain name labels.
+type labelIter struct {
+	orig     string
+	slice    []string
+	curStart int
+	curEnd   int
+	i        int
+}
+
+func (l *labelIter) reset() {
+	l.curStart = 0
+	l.curEnd = 0
+	l.i = 0
+}
+
+func (l *labelIter) done() bool {
+	return l.curStart >= len(l.orig)
+}
+
+func (l *labelIter) result() string {
+	if l.slice != nil {
+		return strings.Join(l.slice, ".")
+	}
+	return l.orig
+}
+
+func (l *labelIter) label() string {
+	if l.slice != nil {
+		return l.slice[l.i]
+	}
+	p := strings.IndexByte(l.orig[l.curStart:], '.')
+	l.curEnd = l.curStart + p
+	if p == -1 {
+		l.curEnd = len(l.orig)
+	}
+	return l.orig[l.curStart:l.curEnd]
+}
+
+// next sets the value to the next label. It skips the last label if it is empty.
+func (l *labelIter) next() {
+	l.i++
+	if l.slice != nil {
+		if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" {
+			l.curStart = len(l.orig)
+		}
+	} else {
+		l.curStart = l.curEnd + 1
+		if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' {
+			l.curStart = len(l.orig)
+		}
+	}
+}
+
+func (l *labelIter) set(s string) {
+	if l.slice == nil {
+		l.slice = strings.Split(l.orig, ".")
+	}
+	l.slice[l.i] = s
+}
+
+// acePrefix is the ASCII Compatible Encoding prefix.
+const acePrefix = "xn--"
+
+func (p *Profile) simplify(cat category) category {
+	switch cat {
+	case disallowedSTD3Mapped:
+		if p.useSTD3Rules {
+			cat = disallowed
+		} else {
+			cat = mapped
+		}
+	case disallowedSTD3Valid:
+		if p.useSTD3Rules {
+			cat = disallowed
+		} else {
+			cat = valid
+		}
+	case deviation:
+		if !p.transitional {
+			cat = valid
+		}
+	case validNV8, validXV8:
+		// TODO: handle V2008
+		cat = valid
+	}
+	return cat
+}
+
+func validateFromPunycode(p *Profile, s string) error {
+	if !norm.NFC.IsNormalString(s) {
+		return &labelError{s, "V1"}
+	}
+	// TODO: detect whether string may have to be normalized in the following
+	// loop.
+	for i := 0; i < len(s); {
+		v, sz := trie.lookupString(s[i:])
+		if sz == 0 {
+			return runeError(utf8.RuneError)
+		}
+		if c := p.simplify(info(v).category()); c != valid && c != deviation {
+			return &labelError{s, "V6"}
+		}
+		i += sz
+	}
+	return nil
+}
+
+const (
+	zwnj = "\u200c"
+	zwj  = "\u200d"
+)
+
+type joinState int8
+
+const (
+	stateStart joinState = iota
+	stateVirama
+	stateBefore
+	stateBeforeVirama
+	stateAfter
+	stateFAIL
+)
+
+var joinStates = [][numJoinTypes]joinState{
+	stateStart: {
+		joiningL:   stateBefore,
+		joiningD:   stateBefore,
+		joinZWNJ:   stateFAIL,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateVirama,
+	},
+	stateVirama: {
+		joiningL: stateBefore,
+		joiningD: stateBefore,
+	},
+	stateBefore: {
+		joiningL:   stateBefore,
+		joiningD:   stateBefore,
+		joiningT:   stateBefore,
+		joinZWNJ:   stateAfter,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateBeforeVirama,
+	},
+	stateBeforeVirama: {
+		joiningL: stateBefore,
+		joiningD: stateBefore,
+		joiningT: stateBefore,
+	},
+	stateAfter: {
+		joiningL:   stateFAIL,
+		joiningD:   stateBefore,
+		joiningT:   stateAfter,
+		joiningR:   stateStart,
+		joinZWNJ:   stateFAIL,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateAfter, // no-op as we can't accept joiners here
+	},
+	stateFAIL: {
+		0:          stateFAIL,
+		joiningL:   stateFAIL,
+		joiningD:   stateFAIL,
+		joiningT:   stateFAIL,
+		joiningR:   stateFAIL,
+		joinZWNJ:   stateFAIL,
+		joinZWJ:    stateFAIL,
+		joinVirama: stateFAIL,
+	},
+}
+
+// validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are
+// already implicitly satisfied by the overall implementation.
+func (p *Profile) validateLabel(s string) (err error) {
+	if s == "" {
+		if p.verifyDNSLength {
+			return &labelError{s, "A4"}
+		}
+		return nil
+	}
+	if !p.validateLabels {
+		return nil
+	}
+	trie := p.trie // p.validateLabels is only set if trie is set.
+	if len(s) > 4 && s[2] == '-' && s[3] == '-' {
+		return &labelError{s, "V2"}
+	}
+	if s[0] == '-' || s[len(s)-1] == '-' {
+		return &labelError{s, "V3"}
+	}
+	// TODO: merge the use of this in the trie.
+	v, sz := trie.lookupString(s)
+	x := info(v)
+	if x.isModifier() {
+		return &labelError{s, "V5"}
+	}
+	// Quickly return in the absence of zero-width (non) joiners.
+	if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 {
+		return nil
+	}
+	st := stateStart
+	for i := 0; ; {
+		jt := x.joinType()
+		if s[i:i+sz] == zwj {
+			jt = joinZWJ
+		} else if s[i:i+sz] == zwnj {
+			jt = joinZWNJ
+		}
+		st = joinStates[st][jt]
+		if x.isViramaModifier() {
+			st = joinStates[st][joinVirama]
+		}
+		if i += sz; i == len(s) {
+			break
+		}
+		v, sz = trie.lookupString(s[i:])
+		x = info(v)
+	}
+	if st == stateFAIL || st == stateAfter {
+		return &labelError{s, "C"}
+	}
+	return nil
+}
+
+func ascii(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= utf8.RuneSelf {
+			return false
+		}
+	}
+	return true
+}

vendor/golang.org/x/net/idna/punycode.go

@@ -0,0 +1,203 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package idna
+
+// This file implements the Punycode algorithm from RFC 3492.
+
+import (
+	"math"
+	"strings"
+	"unicode/utf8"
+)
+
+// These parameter values are specified in section 5.
+//
+// All computation is done with int32s, so that overflow behavior is identical
+// regardless of whether int is 32-bit or 64-bit.
+const (
+	base        int32 = 36
+	damp        int32 = 700
+	initialBias int32 = 72
+	initialN    int32 = 128
+	skew        int32 = 38
+	tmax        int32 = 26
+	tmin        int32 = 1
+)
+
+func punyError(s string) error { return &labelError{s, "A3"} }
+
+// decode decodes a string as specified in section 6.2.
+func decode(encoded string) (string, error) {
+	if encoded == "" {
+		return "", nil
+	}
+	pos := 1 + strings.LastIndex(encoded, "-")
+	if pos == 1 {
+		return "", punyError(encoded)
+	}
+	if pos == len(encoded) {
+		return encoded[:len(encoded)-1], nil
+	}
+	output := make([]rune, 0, len(encoded))
+	if pos != 0 {
+		for _, r := range encoded[:pos-1] {
+			output = append(output, r)
+		}
+	}
+	i, n, bias := int32(0), initialN, initialBias
+	for pos < len(encoded) {
+		oldI, w := i, int32(1)
+		for k := base; ; k += base {
+			if pos == len(encoded) {
+				return "", punyError(encoded)
+			}
+			digit, ok := decodeDigit(encoded[pos])
+			if !ok {
+				return "", punyError(encoded)
+			}
+			pos++
+			i += digit * w
+			if i < 0 {
+				return "", punyError(encoded)
+			}
+			t := k - bias
+			if t < tmin {
+				t = tmin
+			} else if t > tmax {
+				t = tmax
+			}
+			if digit < t {
+				break
+			}
+			w *= base - t
+			if w >= math.MaxInt32/base {
+				return "", punyError(encoded)
+			}
+		}
+		x := int32(len(output) + 1)
+		bias = adapt(i-oldI, x, oldI == 0)
+		n += i / x
+		i %= x
+		if n > utf8.MaxRune || len(output) >= 1024 {
+			return "", punyError(encoded)
+		}
+		output = append(output, 0)
+		copy(output[i+1:], output[i:])
+		output[i] = n
+		i++
+	}
+	return string(output), nil
+}
+
+// encode encodes a string as specified in section 6.3 and prepends prefix to
+// the result.
+//
+// The "while h < length(input)" line in the specification becomes "for
+// remaining != 0" in the Go code, because len(s) in Go is in bytes, not runes.
+func encode(prefix, s string) (string, error) {
+	output := make([]byte, len(prefix), len(prefix)+1+2*len(s))
+	copy(output, prefix)
+	delta, n, bias := int32(0), initialN, initialBias
+	b, remaining := int32(0), int32(0)
+	for _, r := range s {
+		if r < 0x80 {
+			b++
+			output = append(output, byte(r))
+		} else {
+			remaining++
+		}
+	}
+	h := b
+	if b > 0 {
+		output = append(output, '-')
+	}
+	for remaining != 0 {
+		m := int32(0x7fffffff)
+		for _, r := range s {
+			if m > r && r >= n {
+				m = r
+			}
+		}
+		delta += (m - n) * (h + 1)
+		if delta < 0 {
+			return "", punyError(s)
+		}
+		n = m
+		for _, r := range s {
+			if r < n {
+				delta++
+				if delta < 0 {
+					return "", punyError(s)
+				}
+				continue
+			}
+			if r > n {
+				continue
+			}
+			q := delta
+			for k := base; ; k += base {
+				t := k - bias
+				if t < tmin {
+					t = tmin
+				} else if t > tmax {
+					t = tmax
+				}
+				if q < t {
+					break
+				}
+				output = append(output, encodeDigit(t+(q-t)%(base-t)))
+				q = (q - t) / (base - t)
+			}
+			output = append(output, encodeDigit(q))
+			bias = adapt(delta, h+1, h == b)
+			delta = 0
+			h++
+			remaining--
+		}
+		delta++
+		n++
+	}
+	return string(output), nil
+}
+
+func decodeDigit(x byte) (digit int32, ok bool) {
+	switch {
+	case '0' <= x && x <= '9':
+		return int32(x - ('0' - 26)), true
+	case 'A' <= x && x <= 'Z':
+		return int32(x - 'A'), true
+	case 'a' <= x && x <= 'z':
+		return int32(x - 'a'), true
+	}
+	return 0, false
+}
+
+func encodeDigit(digit int32) byte {
+	switch {
+	case 0 <= digit && digit < 26:
+		return byte(digit + 'a')
+	case 26 <= digit && digit < 36:
+		return byte(digit + ('0' - 26))
+	}
+	panic("idna: internal error in punycode encoding")
+}
+
+// adapt is the bias adaptation function specified in section 6.1.
+func adapt(delta, numPoints int32, firstTime bool) int32 {
+	if firstTime {
+		delta /= damp
+	} else {
+		delta /= 2
+	}
+	delta += delta / numPoints
+	k := int32(0)
+	for delta > ((base-tmin)*tmax)/2 {
+		delta /= base - tmin
+		k += base
+	}
+	return k + (base-tmin+1)*delta/(delta+skew)
+}

vendor/golang.org/x/net/idna/trie.go

@@ -0,0 +1,72 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package idna
+
+// appendMapping appends the mapping for the respective rune. isMapped must be
+// true. A mapping is a categorization of a rune as defined in UTS #46.
+func (c info) appendMapping(b []byte, s string) []byte {
+	index := int(c >> indexShift)
+	if c&xorBit == 0 {
+		s := mappings[index:]
+		return append(b, s[1:s[0]+1]...)
+	}
+	b = append(b, s...)
+	if c&inlineXOR == inlineXOR {
+		// TODO: support and handle two-byte inline masks
+		b[len(b)-1] ^= byte(index)
+	} else {
+		for p := len(b) - int(xorData[index]); p < len(b); p++ {
+			index++
+			b[p] ^= xorData[index]
+		}
+	}
+	return b
+}
+
+// Sparse block handling code.
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var idnaSparse = sparseBlocks{
+	values: idnaSparseValues[:],
+	offset: idnaSparseOffset[:],
+}
+
+// Don't use newIdnaTrie to avoid unconditional linking in of the table.
+var trie = &idnaTrie{}
+
+// lookup determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

vendor/golang.org/x/net/idna/trieval.go

@@ -0,0 +1,119 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package idna
+
+// This file contains definitions for interpreting the trie value of the idna
+// trie generated by "go run gen*.go". It is shared by both the generator
+// program and the resultant package. Sharing is achieved by the generator
+// copying gen_trieval.go to trieval.go and changing what's above this comment.
+
+// info holds information from the IDNA mapping table for a single rune. It is
+// the value returned by a trie lookup. In most cases, all information fits in
+// a 16-bit value. For mappings, this value may contain an index into a slice
+// with the mapped string. Such mappings can consist of the actual mapped value
+// or an XOR pattern to be applied to the bytes of the UTF8 encoding of the
+// input rune. This technique is used by the cases packages and reduces the
+// table size significantly.
+//
+// The per-rune values have the following format:
+//
+//   if mapped {
+//     if inlinedXOR {
+//       15..13 inline XOR marker
+//       12..11 unused
+//       10..3  inline XOR mask
+//     } else {
+//       15..3  index into xor or mapping table
+//     }
+//   } else {
+//       15..14 unused
+//       13     mayNeedNorm
+//       12..11 attributes
+//       10..8  joining type
+//        7..3  category type
+//   }
+//      2  use xor pattern
+//   1..0  mapped category
+//
+// See the definitions below for a more detailed description of the various
+// bits.
+type info uint16
+
+const (
+	catSmallMask = 0x3
+	catBigMask   = 0xF8
+	indexShift   = 3
+	xorBit       = 0x4    // interpret the index as an xor pattern
+	inlineXOR    = 0xE000 // These bits are set if the XOR pattern is inlined.
+
+	joinShift = 8
+	joinMask  = 0x07
+
+	// Attributes
+	attributesMask = 0x1800
+	viramaModifier = 0x1800
+	modifier       = 0x1000
+	rtl            = 0x0800
+
+	mayNeedNorm = 0x2000
+)
+
+// A category corresponds to a category defined in the IDNA mapping table.
+type category uint16
+
+const (
+	unknown              category = 0 // not currently defined in unicode.
+	mapped               category = 1
+	disallowedSTD3Mapped category = 2
+	deviation            category = 3
+)
+
+const (
+	valid               category = 0x08
+	validNV8            category = 0x18
+	validXV8            category = 0x28
+	disallowed          category = 0x40
+	disallowedSTD3Valid category = 0x80
+	ignored             category = 0xC0
+)
+
+// join types and additional rune information
+const (
+	joiningL = (iota + 1)
+	joiningD
+	joiningT
+	joiningR
+
+	//the following types are derived during processing
+	joinZWJ
+	joinZWNJ
+	joinVirama
+	numJoinTypes
+)
+
+func (c info) isMapped() bool {
+	return c&0x3 != 0
+}
+
+func (c info) category() category {
+	small := c & catSmallMask
+	if small != 0 {
+		return category(small)
+	}
+	return category(c & catBigMask)
+}
+
+func (c info) joinType() info {
+	if c.isMapped() {
+		return 0
+	}
+	return (c >> joinShift) & joinMask
+}
+
+func (c info) isModifier() bool {
+	return c&(modifier|catSmallMask) == modifier
+}
+
+func (c info) isViramaModifier() bool {
+	return c&(attributesMask|catSmallMask) == viramaModifier
+}

vendor/golang.org/x/text/secure/bidirule/bidirule.go

@@ -0,0 +1,336 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bidirule implements the Bidi Rule defined by RFC 5893.
+//
+// This package is under development. The API may change without notice and
+// without preserving backward compatibility.
+package bidirule
+
+import (
+	"errors"
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/bidi"
+)
+
+// This file contains an implementation of RFC 5893: Right-to-Left Scripts for
+// Internationalized Domain Names for Applications (IDNA)
+//
+// A label is an individual component of a domain name.  Labels are usually
+// shown separated by dots; for example, the domain name "www.example.com" is
+// composed of three labels: "www", "example", and "com".
+//
+// An RTL label is a label that contains at least one character of class R, AL,
+// or AN. An LTR label is any label that is not an RTL label.
+//
+// A "Bidi domain name" is a domain name that contains at least one RTL label.
+//
+//  The following guarantees can be made based on the above:
+//
+//  o  In a domain name consisting of only labels that satisfy the rule,
+//     the requirements of Section 3 are satisfied.  Note that even LTR
+//     labels and pure ASCII labels have to be tested.
+//
+//  o  In a domain name consisting of only LDH labels (as defined in the
+//     Definitions document [RFC5890]) and labels that satisfy the rule,
+//     the requirements of Section 3 are satisfied as long as a label
+//     that starts with an ASCII digit does not come after a
+//     right-to-left label.
+//
+//  No guarantee is given for other combinations.
+
+// ErrInvalid indicates a label is invalid according to the Bidi Rule.
+var ErrInvalid = errors.New("bidirule: failed Bidi Rule")
+
+type ruleState uint8
+
+const (
+	ruleInitial ruleState = iota
+	ruleLTR
+	ruleLTRFinal
+	ruleRTL
+	ruleRTLFinal
+	ruleInvalid
+)
+
+type ruleTransition struct {
+	next ruleState
+	mask uint16
+}
+
+var transitions = [...][2]ruleTransition{
+	// [2.1] The first character must be a character with Bidi property L, R, or
+	// AL. If it has the R or AL property, it is an RTL label; if it has the L
+	// property, it is an LTR label.
+	ruleInitial: {
+		{ruleLTRFinal, 1 << bidi.L},
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL},
+	},
+	ruleRTL: {
+		// [2.3] In an RTL label, the end of the label must be a character with
+		// Bidi property R, AL, EN, or AN, followed by zero or more characters
+		// with Bidi property NSM.
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN},
+
+		// [2.2] In an RTL label, only characters with the Bidi properties R,
+		// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.3]
+		{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
+	},
+	ruleRTLFinal: {
+		// [2.3] In an RTL label, the end of the label must be a character with
+		// Bidi property R, AL, EN, or AN, followed by zero or more characters
+		// with Bidi property NSM.
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN | 1<<bidi.NSM},
+
+		// [2.2] In an RTL label, only characters with the Bidi properties R,
+		// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.3] and NSM.
+		{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
+	},
+	ruleLTR: {
+		// [2.6] In an LTR label, the end of the label must be a character with
+		// Bidi property L or EN, followed by zero or more characters with Bidi
+		// property NSM.
+		{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN},
+
+		// [2.5] In an LTR label, only characters with the Bidi properties L,
+		// EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.6].
+		{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
+	},
+	ruleLTRFinal: {
+		// [2.6] In an LTR label, the end of the label must be a character with
+		// Bidi property L or EN, followed by zero or more characters with Bidi
+		// property NSM.
+		{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN | 1<<bidi.NSM},
+
+		// [2.5] In an LTR label, only characters with the Bidi properties L,
+		// EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.6].
+		{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
+	},
+	ruleInvalid: {
+		{ruleInvalid, 0},
+		{ruleInvalid, 0},
+	},
+}
+
+// [2.4] In an RTL label, if an EN is present, no AN may be present, and
+// vice versa.
+const exclusiveRTL = uint16(1<<bidi.EN | 1<<bidi.AN)
+
+// From RFC 5893
+// An RTL label is a label that contains at least one character of type
+// R, AL, or AN.
+//
+// An LTR label is any label that is not an RTL label.
+
+// Direction reports the direction of the given label as defined by RFC 5893.
+// The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func Direction(b []byte) bidi.Direction {
+	for i := 0; i < len(b); {
+		e, sz := bidi.Lookup(b[i:])
+		if sz == 0 {
+			i++
+		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
+	}
+	return bidi.LeftToRight
+}
+
+// DirectionString reports the direction of the given label as defined by RFC
+// 5893. The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func DirectionString(s string) bidi.Direction {
+	for i := 0; i < len(s); {
+		e, sz := bidi.LookupString(s[i:])
+		if sz == 0 {
+			i++
+			continue
+		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
+	}
+	return bidi.LeftToRight
+}
+
+// Valid reports whether b conforms to the BiDi rule.
+func Valid(b []byte) bool {
+	var t Transformer
+	if n, ok := t.advance(b); !ok || n < len(b) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// ValidString reports whether s conforms to the BiDi rule.
+func ValidString(s string) bool {
+	var t Transformer
+	if n, ok := t.advanceString(s); !ok || n < len(s) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// New returns a Transformer that verifies that input adheres to the Bidi Rule.
+func New() *Transformer {
+	return &Transformer{}
+}
+
+// Transformer implements transform.Transform.
+type Transformer struct {
+	state  ruleState
+	hasRTL bool
+	seen   uint16
+}
+
+// A rule can only be violated for "Bidi Domain names", meaning if one of the
+// following categories has been observed.
+func (t *Transformer) isRTL() bool {
+	const isRTL = 1<<bidi.R | 1<<bidi.AL | 1<<bidi.AN
+	return t.seen&isRTL != 0
+}
+
+// Reset implements transform.Transformer.
+func (t *Transformer) Reset() { *t = Transformer{} }
+
+// Transform implements transform.Transformer. This Transformer has state and
+// needs to be reset between uses.
+func (t *Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if len(dst) < len(src) {
+		src = src[:len(dst)]
+		atEOF = false
+		err = transform.ErrShortDst
+	}
+	n, err1 := t.Span(src, atEOF)
+	copy(dst, src[:n])
+	if err == nil || err1 != nil && err1 != transform.ErrShortSrc {
+		err = err1
+	}
+	return n, n, err
+}
+
+// Span returns the first n bytes of src that conform to the Bidi rule.
+func (t *Transformer) Span(src []byte, atEOF bool) (n int, err error) {
+	if t.state == ruleInvalid && t.isRTL() {
+		return 0, ErrInvalid
+	}
+	n, ok := t.advance(src)
+	switch {
+	case !ok:
+		err = ErrInvalid
+	case n < len(src):
+		if !atEOF {
+			err = transform.ErrShortSrc
+			break
+		}
+		err = ErrInvalid
+	case !t.isFinal():
+		err = ErrInvalid
+	}
+	return n, err
+}
+
+// Precomputing the ASCII values decreases running time for the ASCII fast path
+// by about 30%.
+var asciiTable [128]bidi.Properties
+
+func init() {
+	for i := range asciiTable {
+		p, _ := bidi.LookupRune(rune(i))
+		asciiTable[i] = p
+	}
+}
+
+func (t *Transformer) advance(s []byte) (n int, ok bool) {
+	var e bidi.Properties
+	var sz int
+	for n < len(s) {
+		if s[n] < utf8.RuneSelf {
+			e, sz = asciiTable[s[n]], 1
+		} else {
+			e, sz = bidi.Lookup(s[n:])
+			if sz <= 1 {
+				if sz == 1 {
+					// We always consider invalid UTF-8 to be invalid, even if
+					// the string has not yet been determined to be RTL.
+					// TODO: is this correct?
+					return n, false
+				}
+				return n, true // incomplete UTF-8 encoding
+			}
+		}
+		// TODO: using CompactClass would result in noticeable speedup.
+		// See unicode/bidi/prop.go:Properties.CompactClass.
+		c := uint16(1 << e.Class())
+		t.seen |= c
+		if t.seen&exclusiveRTL == exclusiveRTL {
+			t.state = ruleInvalid
+			return n, false
+		}
+		switch tr := transitions[t.state]; {
+		case tr[0].mask&c != 0:
+			t.state = tr[0].next
+		case tr[1].mask&c != 0:
+			t.state = tr[1].next
+		default:
+			t.state = ruleInvalid
+			if t.isRTL() {
+				return n, false
+			}
+		}
+		n += sz
+	}
+	return n, true
+}
+
+func (t *Transformer) advanceString(s string) (n int, ok bool) {
+	var e bidi.Properties
+	var sz int
+	for n < len(s) {
+		if s[n] < utf8.RuneSelf {
+			e, sz = asciiTable[s[n]], 1
+		} else {
+			e, sz = bidi.LookupString(s[n:])
+			if sz <= 1 {
+				if sz == 1 {
+					return n, false // invalid UTF-8
+				}
+				return n, true // incomplete UTF-8 encoding
+			}
+		}
+		// TODO: using CompactClass results in noticeable speedup.
+		// See unicode/bidi/prop.go:Properties.CompactClass.
+		c := uint16(1 << e.Class())
+		t.seen |= c
+		if t.seen&exclusiveRTL == exclusiveRTL {
+			t.state = ruleInvalid
+			return n, false
+		}
+		switch tr := transitions[t.state]; {
+		case tr[0].mask&c != 0:
+			t.state = tr[0].next
+		case tr[1].mask&c != 0:
+			t.state = tr[1].next
+		default:
+			t.state = ruleInvalid
+			if t.isRTL() {
+				return n, false
+			}
+		}
+		n += sz
+	}
+	return n, true
+}

vendor/golang.org/x/text/secure/bidirule/bidirule10.0.0.go

@@ -0,0 +1,11 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.10
+
+package bidirule
+
+func (t *Transformer) isFinal() bool {
+	return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
+}

vendor/golang.org/x/text/secure/bidirule/bidirule9.0.0.go

@@ -0,0 +1,14 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.10
+
+package bidirule
+
+func (t *Transformer) isFinal() bool {
+	if !t.isRTL() {
+		return true
+	}
+	return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
+}

vendor/golang.org/x/text/unicode/bidi/bidi.go

@@ -0,0 +1,198 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go gen_trieval.go gen_ranges.go
+
+// Package bidi contains functionality for bidirectional text support.
+//
+// See http://www.unicode.org/reports/tr9.
+//
+// NOTE: UNDER CONSTRUCTION. This API may change in backwards incompatible ways
+// and without notice.
+package bidi // import "golang.org/x/text/unicode/bidi"
+
+// TODO:
+// The following functionality would not be hard to implement, but hinges on
+// the definition of a Segmenter interface. For now this is up to the user.
+// - Iterate over paragraphs
+// - Segmenter to iterate over runs directly from a given text.
+// Also:
+// - Transformer for reordering?
+// - Transformer (validator, really) for Bidi Rule.
+
+// This API tries to avoid dealing with embedding levels for now. Under the hood
+// these will be computed, but the question is to which extent the user should
+// know they exist. We should at some point allow the user to specify an
+// embedding hierarchy, though.
+
+// A Direction indicates the overall flow of text.
+type Direction int
+
+const (
+	// LeftToRight indicates the text contains no right-to-left characters and
+	// that either there are some left-to-right characters or the option
+	// DefaultDirection(LeftToRight) was passed.
+	LeftToRight Direction = iota
+
+	// RightToLeft indicates the text contains no left-to-right characters and
+	// that either there are some right-to-left characters or the option
+	// DefaultDirection(RightToLeft) was passed.
+	RightToLeft
+
+	// Mixed indicates text contains both left-to-right and right-to-left
+	// characters.
+	Mixed
+
+	// Neutral means that text contains no left-to-right and right-to-left
+	// characters and that no default direction has been set.
+	Neutral
+)
+
+type options struct{}
+
+// An Option is an option for Bidi processing.
+type Option func(*options)
+
+// ICU allows the user to define embedding levels. This may be used, for example,
+// to use hierarchical structure of markup languages to define embeddings.
+// The following option may be a way to expose this functionality in this API.
+// // LevelFunc sets a function that associates nesting levels with the given text.
+// // The levels function will be called with monotonically increasing values for p.
+// func LevelFunc(levels func(p int) int) Option {
+// 	panic("unimplemented")
+// }
+
+// DefaultDirection sets the default direction for a Paragraph. The direction is
+// overridden if the text contains directional characters.
+func DefaultDirection(d Direction) Option {
+	panic("unimplemented")
+}
+
+// A Paragraph holds a single Paragraph for Bidi processing.
+type Paragraph struct {
+	// buffers
+}
+
+// SetBytes configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// SetString configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// IsLeftToRight reports whether the principle direction of rendering for this
+// paragraphs is left-to-right. If this returns false, the principle direction
+// of rendering is right-to-left.
+func (p *Paragraph) IsLeftToRight() bool {
+	panic("unimplemented")
+}
+
+// Direction returns the direction of the text of this paragraph.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (p *Paragraph) Direction() Direction {
+	panic("unimplemented")
+}
+
+// RunAt reports the Run at the given position of the input text.
+//
+// This method can be used for computing line breaks on paragraphs.
+func (p *Paragraph) RunAt(pos int) Run {
+	panic("unimplemented")
+}
+
+// Order computes the visual ordering of all the runs in a Paragraph.
+func (p *Paragraph) Order() (Ordering, error) {
+	panic("unimplemented")
+}
+
+// Line computes the visual ordering of runs for a single line starting and
+// ending at the given positions in the original text.
+func (p *Paragraph) Line(start, end int) (Ordering, error) {
+	panic("unimplemented")
+}
+
+// An Ordering holds the computed visual order of runs of a Paragraph. Calling
+// SetBytes or SetString on the originating Paragraph invalidates an Ordering.
+// The methods of an Ordering should only be called by one goroutine at a time.
+type Ordering struct{}
+
+// Direction reports the directionality of the runs.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (o *Ordering) Direction() Direction {
+	panic("unimplemented")
+}
+
+// NumRuns returns the number of runs.
+func (o *Ordering) NumRuns() int {
+	panic("unimplemented")
+}
+
+// Run returns the ith run within the ordering.
+func (o *Ordering) Run(i int) Run {
+	panic("unimplemented")
+}
+
+// TODO: perhaps with options.
+// // Reorder creates a reader that reads the runes in visual order per character.
+// // Modifiers remain after the runes they modify.
+// func (l *Runs) Reorder() io.Reader {
+// 	panic("unimplemented")
+// }
+
+// A Run is a continuous sequence of characters of a single direction.
+type Run struct {
+}
+
+// String returns the text of the run in its original order.
+func (r *Run) String() string {
+	panic("unimplemented")
+}
+
+// Bytes returns the text of the run in its original order.
+func (r *Run) Bytes() []byte {
+	panic("unimplemented")
+}
+
+// TODO: methods for
+// - Display order
+// - headers and footers
+// - bracket replacement.
+
+// Direction reports the direction of the run.
+func (r *Run) Direction() Direction {
+	panic("unimplemented")
+}
+
+// Position of the Run within the text passed to SetBytes or SetString of the
+// originating Paragraph value.
+func (r *Run) Pos() (start, end int) {
+	panic("unimplemented")
+}
+
+// AppendReverse reverses the order of characters of in, appends them to out,
+// and returns the result. Modifiers will still follow the runes they modify.
+// Brackets are replaced with their counterparts.
+func AppendReverse(out, in []byte) []byte {
+	panic("unimplemented")
+}
+
+// ReverseString reverses the order of characters in s and returns a new string.
+// Modifiers will still follow the runes they modify. Brackets are replaced with
+// their counterparts.
+func ReverseString(s string) string {
+	panic("unimplemented")
+}

vendor/golang.org/x/text/unicode/bidi/bracket.go

@@ -0,0 +1,335 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import (
+	"container/list"
+	"fmt"
+	"sort"
+)
+
+// This file contains a port of the reference implementation of the
+// Bidi Parentheses Algorithm:
+// http://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/BidiPBAReference.java
+//
+// The implementation in this file covers definitions BD14-BD16 and rule N0
+// of UAX#9.
+//
+// Some preprocessing is done for each rune before data is passed to this
+// algorithm:
+//  - opening and closing brackets are identified
+//  - a bracket pair type, like '(' and ')' is assigned a unique identifier that
+//    is identical for the opening and closing bracket. It is left to do these
+//    mappings.
+//  - The BPA algorithm requires that bracket characters that are canonical
+//    equivalents of each other be able to be substituted for each other.
+//    It is the responsibility of the caller to do this canonicalization.
+//
+// In implementing BD16, this implementation departs slightly from the "logical"
+// algorithm defined in UAX#9. In particular, the stack referenced there
+// supports operations that go beyond a "basic" stack. An equivalent
+// implementation based on a linked list is used here.
+
+// Bidi_Paired_Bracket_Type
+// BD14. An opening paired bracket is a character whose
+// Bidi_Paired_Bracket_Type property value is Open.
+//
+// BD15. A closing paired bracket is a character whose
+// Bidi_Paired_Bracket_Type property value is Close.
+type bracketType byte
+
+const (
+	bpNone bracketType = iota
+	bpOpen
+	bpClose
+)
+
+// bracketPair holds a pair of index values for opening and closing bracket
+// location of a bracket pair.
+type bracketPair struct {
+	opener int
+	closer int
+}
+
+func (b *bracketPair) String() string {
+	return fmt.Sprintf("(%v, %v)", b.opener, b.closer)
+}
+
+// bracketPairs is a slice of bracketPairs with a sort.Interface implementation.
+type bracketPairs []bracketPair
+
+func (b bracketPairs) Len() int           { return len(b) }
+func (b bracketPairs) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b bracketPairs) Less(i, j int) bool { return b[i].opener < b[j].opener }
+
+// resolvePairedBrackets runs the paired bracket part of the UBA algorithm.
+//
+// For each rune, it takes the indexes into the original string, the class the
+// bracket type (in pairTypes) and the bracket identifier (pairValues). It also
+// takes the direction type for the start-of-sentence and the embedding level.
+//
+// The identifiers for bracket types are the rune of the canonicalized opening
+// bracket for brackets (open or close) or 0 for runes that are not brackets.
+func resolvePairedBrackets(s *isolatingRunSequence) {
+	p := bracketPairer{
+		sos:              s.sos,
+		openers:          list.New(),
+		codesIsolatedRun: s.types,
+		indexes:          s.indexes,
+	}
+	dirEmbed := L
+	if s.level&1 != 0 {
+		dirEmbed = R
+	}
+	p.locateBrackets(s.p.pairTypes, s.p.pairValues)
+	p.resolveBrackets(dirEmbed, s.p.initialTypes)
+}
+
+type bracketPairer struct {
+	sos Class // direction corresponding to start of sequence
+
+	// The following is a restatement of BD 16 using non-algorithmic language.
+	//
+	// A bracket pair is a pair of characters consisting of an opening
+	// paired bracket and a closing paired bracket such that the
+	// Bidi_Paired_Bracket property value of the former equals the latter,
+	// subject to the following constraints.
+	// - both characters of a pair occur in the same isolating run sequence
+	// - the closing character of a pair follows the opening character
+	// - any bracket character can belong at most to one pair, the earliest possible one
+	// - any bracket character not part of a pair is treated like an ordinary character
+	// - pairs may nest properly, but their spans may not overlap otherwise
+
+	// Bracket characters with canonical decompositions are supposed to be
+	// treated as if they had been normalized, to allow normalized and non-
+	// normalized text to give the same result. In this implementation that step
+	// is pushed out to the caller. The caller has to ensure that the pairValue
+	// slices contain the rune of the opening bracket after normalization for
+	// any opening or closing bracket.
+
+	openers *list.List // list of positions for opening brackets
+
+	// bracket pair positions sorted by location of opening bracket
+	pairPositions bracketPairs
+
+	codesIsolatedRun []Class // directional bidi codes for an isolated run
+	indexes          []int   // array of index values into the original string
+
+}
+
+// matchOpener reports whether characters at given positions form a matching
+// bracket pair.
+func (p *bracketPairer) matchOpener(pairValues []rune, opener, closer int) bool {
+	return pairValues[p.indexes[opener]] == pairValues[p.indexes[closer]]
+}
+
+const maxPairingDepth = 63
+
+// locateBrackets locates matching bracket pairs according to BD16.
+//
+// This implementation uses a linked list instead of a stack, because, while
+// elements are added at the front (like a push) they are not generally removed
+// in atomic 'pop' operations, reducing the benefit of the stack archetype.
+func (p *bracketPairer) locateBrackets(pairTypes []bracketType, pairValues []rune) {
+	// traverse the run
+	// do that explicitly (not in a for-each) so we can record position
+	for i, index := range p.indexes {
+
+		// look at the bracket type for each character
+		if pairTypes[index] == bpNone || p.codesIsolatedRun[i] != ON {
+			// continue scanning
+			continue
+		}
+		switch pairTypes[index] {
+		case bpOpen:
+			// check if maximum pairing depth reached
+			if p.openers.Len() == maxPairingDepth {
+				p.openers.Init()
+				return
+			}
+			// remember opener location, most recent first
+			p.openers.PushFront(i)
+
+		case bpClose:
+			// see if there is a match
+			count := 0
+			for elem := p.openers.Front(); elem != nil; elem = elem.Next() {
+				count++
+				opener := elem.Value.(int)
+				if p.matchOpener(pairValues, opener, i) {
+					// if the opener matches, add nested pair to the ordered list
+					p.pairPositions = append(p.pairPositions, bracketPair{opener, i})
+					// remove up to and including matched opener
+					for ; count > 0; count-- {
+						p.openers.Remove(p.openers.Front())
+					}
+					break
+				}
+			}
+			sort.Sort(p.pairPositions)
+			// if we get here, the closing bracket matched no openers
+			// and gets ignored
+		}
+	}
+}
+
+// Bracket pairs within an isolating run sequence are processed as units so
+// that both the opening and the closing paired bracket in a pair resolve to
+// the same direction.
+//
+// N0. Process bracket pairs in an isolating run sequence sequentially in
+// the logical order of the text positions of the opening paired brackets
+// using the logic given below. Within this scope, bidirectional types EN
+// and AN are treated as R.
+//
+// Identify the bracket pairs in the current isolating run sequence
+// according to BD16. For each bracket-pair element in the list of pairs of
+// text positions:
+//
+// a Inspect the bidirectional types of the characters enclosed within the
+// bracket pair.
+//
+// b If any strong type (either L or R) matching the embedding direction is
+// found, set the type for both brackets in the pair to match the embedding
+// direction.
+//
+// o [ e ] o -> o e e e o
+//
+// o [ o e ] -> o e o e e
+//
+// o [ NI e ] -> o e NI e e
+//
+// c Otherwise, if a strong type (opposite the embedding direction) is
+// found, test for adjacent strong types as follows: 1 First, check
+// backwards before the opening paired bracket until the first strong type
+// (L, R, or sos) is found. If that first preceding strong type is opposite
+// the embedding direction, then set the type for both brackets in the pair
+// to that type. 2 Otherwise, set the type for both brackets in the pair to
+// the embedding direction.
+//
+// o [ o ] e -> o o o o e
+//
+// o [ o NI ] o -> o o o NI o o
+//
+// e [ o ] o -> e e o e o
+//
+// e [ o ] e -> e e o e e
+//
+// e ( o [ o ] NI ) e -> e e o o o o NI e e
+//
+// d Otherwise, do not set the type for the current bracket pair. Note that
+// if the enclosed text contains no strong types the paired brackets will
+// both resolve to the same level when resolved individually using rules N1
+// and N2.
+//
+// e ( NI ) o -> e ( NI ) o
+
+// getStrongTypeN0 maps character's directional code to strong type as required
+// by rule N0.
+//
+// TODO: have separate type for "strong" directionality.
+func (p *bracketPairer) getStrongTypeN0(index int) Class {
+	switch p.codesIsolatedRun[index] {
+	// in the scope of N0, number types are treated as R
+	case EN, AN, AL, R:
+		return R
+	case L:
+		return L
+	default:
+		return ON
+	}
+}
+
+// classifyPairContent reports the strong types contained inside a Bracket Pair,
+// assuming the given embedding direction.
+//
+// It returns ON if no strong type is found. If a single strong type is found,
+// it returns this this type. Otherwise it returns the embedding direction.
+//
+// TODO: use separate type for "strong" directionality.
+func (p *bracketPairer) classifyPairContent(loc bracketPair, dirEmbed Class) Class {
+	dirOpposite := ON
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		dir := p.getStrongTypeN0(i)
+		if dir == ON {
+			continue
+		}
+		if dir == dirEmbed {
+			return dir // type matching embedding direction found
+		}
+		dirOpposite = dir
+	}
+	// return ON if no strong type found, or class opposite to dirEmbed
+	return dirOpposite
+}
+
+// classBeforePair determines which strong types are present before a Bracket
+// Pair. Return R or L if strong type found, otherwise ON.
+func (p *bracketPairer) classBeforePair(loc bracketPair) Class {
+	for i := loc.opener - 1; i >= 0; i-- {
+		if dir := p.getStrongTypeN0(i); dir != ON {
+			return dir
+		}
+	}
+	// no strong types found, return sos
+	return p.sos
+}
+
+// assignBracketType implements rule N0 for a single bracket pair.
+func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) {
+	// rule "N0, a", inspect contents of pair
+	dirPair := p.classifyPairContent(loc, dirEmbed)
+
+	// dirPair is now L, R, or N (no strong type found)
+
+	// the following logical tests are performed out of order compared to
+	// the statement of the rules but yield the same results
+	if dirPair == ON {
+		return // case "d" - nothing to do
+	}
+
+	if dirPair != dirEmbed {
+		// case "c": strong type found, opposite - check before (c.1)
+		dirPair = p.classBeforePair(loc)
+		if dirPair == dirEmbed || dirPair == ON {
+			// no strong opposite type found before - use embedding (c.2)
+			dirPair = dirEmbed
+		}
+	}
+	// else: case "b", strong type found matching embedding,
+	// no explicit action needed, as dirPair is already set to embedding
+	// direction
+
+	// set the bracket types to the type found
+	p.setBracketsToType(loc, dirPair, initialTypes)
+}
+
+func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) {
+	p.codesIsolatedRun[loc.opener] = dirPair
+	p.codesIsolatedRun[loc.closer] = dirPair
+
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+
+	for i := loc.closer + 1; i < len(p.indexes); i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+}
+
+// resolveBrackets implements rule N0 for a list of pairs.
+func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) {
+	for _, loc := range p.pairPositions {
+		p.assignBracketType(loc, dirEmbed, initialTypes)
+	}
+}

vendor/golang.org/x/text/unicode/bidi/prop.go

@@ -0,0 +1,206 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import "unicode/utf8"
+
+// Properties provides access to BiDi properties of runes.
+type Properties struct {
+	entry uint8
+	last  uint8
+}
+
+var trie = newBidiTrie(0)
+
+// TODO: using this for bidirule reduces the running time by about 5%. Consider
+// if this is worth exposing or if we can find a way to speed up the Class
+// method.
+//
+// // CompactClass is like Class, but maps all of the BiDi control classes
+// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control.
+// func (p Properties) CompactClass() Class {
+// 	return Class(p.entry & 0x0F)
+// }
+
+// Class returns the Bidi class for p.
+func (p Properties) Class() Class {
+	c := Class(p.entry & 0x0F)
+	if c == Control {
+		c = controlByteToClass[p.last&0xF]
+	}
+	return c
+}
+
+// IsBracket reports whether the rune is a bracket.
+func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 }
+
+// IsOpeningBracket reports whether the rune is an opening bracket.
+// IsBracket must return true.
+func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 }
+
+// TODO: find a better API and expose.
+func (p Properties) reverseBracket(r rune) rune {
+	return xorMasks[p.entry>>xorMaskShift] ^ r
+}
+
+var controlByteToClass = [16]Class{
+	0xD: LRO, // U+202D LeftToRightOverride,
+	0xE: RLO, // U+202E RightToLeftOverride,
+	0xA: LRE, // U+202A LeftToRightEmbedding,
+	0xB: RLE, // U+202B RightToLeftEmbedding,
+	0xC: PDF, // U+202C PopDirectionalFormat,
+	0x6: LRI, // U+2066 LeftToRightIsolate,
+	0x7: RLI, // U+2067 RightToLeftIsolate,
+	0x8: FSI, // U+2068 FirstStrongIsolate,
+	0x9: PDI, // U+2069 PopDirectionalIsolate,
+}
+
+// LookupRune returns properties for r.
+func LookupRune(r rune) (p Properties, size int) {
+	var buf [4]byte
+	n := utf8.EncodeRune(buf[:], r)
+	return Lookup(buf[:n])
+}
+
+// TODO: these lookup methods are based on the generated trie code. The returned
+// sizes have slightly different semantics from the generated code, in that it
+// always returns size==1 for an illegal UTF-8 byte (instead of the length
+// of the maximum invalid subsequence). Most Transformers, like unicode/norm,
+// leave invalid UTF-8 untouched, in which case it has performance benefits to
+// do so (without changing the semantics). Bidi requires the semantics used here
+// for the bidirule implementation to be compatible with the Go semantics.
+//  They ultimately should perhaps be adopted by all trie implementations, for
+// convenience sake.
+// This unrolled code also boosts performance of the secure/bidirule package by
+// about 30%.
+// So, to remove this code:
+//   - add option to trie generator to define return type.
+//   - always return 1 byte size for ill-formed UTF-8 runes.
+
+// Lookup returns properties for the first rune in s and the width in bytes of
+// its encoding. The size will be 0 if s does not hold enough bytes to complete
+// the encoding.
+func Lookup(s []byte) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}
+
+// LookupString returns properties for the first rune in s and the width in
+// bytes of its encoding. The size will be 0 if s does not hold enough bytes to
+// complete the encoding.
+func LookupString(s string) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}

vendor/golang.org/x/text/unicode/bidi/trieval.go

@@ -0,0 +1,60 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package bidi
+
+// Class is the Unicode BiDi class. Each rune has a single class.
+type Class uint
+
+const (
+	L       Class = iota // LeftToRight
+	R                    // RightToLeft
+	EN                   // EuropeanNumber
+	ES                   // EuropeanSeparator
+	ET                   // EuropeanTerminator
+	AN                   // ArabicNumber
+	CS                   // CommonSeparator
+	B                    // ParagraphSeparator
+	S                    // SegmentSeparator
+	WS                   // WhiteSpace
+	ON                   // OtherNeutral
+	BN                   // BoundaryNeutral
+	NSM                  // NonspacingMark
+	AL                   // ArabicLetter
+	Control              // Control LRO - PDI
+
+	numClass
+
+	LRO // LeftToRightOverride
+	RLO // RightToLeftOverride
+	LRE // LeftToRightEmbedding
+	RLE // RightToLeftEmbedding
+	PDF // PopDirectionalFormat
+	LRI // LeftToRightIsolate
+	RLI // RightToLeftIsolate
+	FSI // FirstStrongIsolate
+	PDI // PopDirectionalIsolate
+
+	unknownClass = ^Class(0)
+)
+
+var controlToClass = map[rune]Class{
+	0x202D: LRO, // LeftToRightOverride,
+	0x202E: RLO, // RightToLeftOverride,
+	0x202A: LRE, // LeftToRightEmbedding,
+	0x202B: RLE, // RightToLeftEmbedding,
+	0x202C: PDF, // PopDirectionalFormat,
+	0x2066: LRI, // LeftToRightIsolate,
+	0x2067: RLI, // RightToLeftIsolate,
+	0x2068: FSI, // FirstStrongIsolate,
+	0x2069: PDI, // PopDirectionalIsolate,
+}
+
+// A trie entry has the following bits:
+// 7..5  XOR mask for brackets
+// 4     1: Bracket open, 0: Bracket close
+// 3..0  Class type
+
+const (
+	openMask     = 0x10
+	xorMaskShift = 5
+)

vendor/golang.org/x/text/unicode/norm/composition.go

@@ -0,0 +1,508 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+const (
+	maxNonStarters = 30
+	// The maximum number of characters needed for a buffer is
+	// maxNonStarters + 1 for the starter + 1 for the GCJ
+	maxBufferSize    = maxNonStarters + 2
+	maxNFCExpansion  = 3  // NFC(0x1D160)
+	maxNFKCExpansion = 18 // NFKC(0xFDFA)
+
+	maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
+)
+
+// ssState is used for reporting the segment state after inserting a rune.
+// It is returned by streamSafe.next.
+type ssState int
+
+const (
+	// Indicates a rune was successfully added to the segment.
+	ssSuccess ssState = iota
+	// Indicates a rune starts a new segment and should not be added.
+	ssStarter
+	// Indicates a rune caused a segment overflow and a CGJ should be inserted.
+	ssOverflow
+)
+
+// streamSafe implements the policy of when a CGJ should be inserted.
+type streamSafe uint8
+
+// first inserts the first rune of a segment. It is a faster version of next if
+// it is known p represents the first rune in a segment.
+func (ss *streamSafe) first(p Properties) {
+	*ss = streamSafe(p.nTrailingNonStarters())
+}
+
+// insert returns a ssState value to indicate whether a rune represented by p
+// can be inserted.
+func (ss *streamSafe) next(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	n := p.nLeadingNonStarters()
+	if *ss += streamSafe(n); *ss > maxNonStarters {
+		*ss = 0
+		return ssOverflow
+	}
+	// The Stream-Safe Text Processing prescribes that the counting can stop
+	// as soon as a starter is encountered. However, there are some starters,
+	// like Jamo V and T, that can combine with other runes, leaving their
+	// successive non-starters appended to the previous, possibly causing an
+	// overflow. We will therefore consider any rune with a non-zero nLead to
+	// be a non-starter. Note that it always hold that if nLead > 0 then
+	// nLead == nTrail.
+	if n == 0 {
+		*ss = streamSafe(p.nTrailingNonStarters())
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+// backwards is used for checking for overflow and segment starts
+// when traversing a string backwards. Users do not need to call first
+// for the first rune. The state of the streamSafe retains the count of
+// the non-starters loaded.
+func (ss *streamSafe) backwards(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	c := *ss + streamSafe(p.nTrailingNonStarters())
+	if c > maxNonStarters {
+		return ssOverflow
+	}
+	*ss = c
+	if p.nLeadingNonStarters() == 0 {
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+func (ss streamSafe) isMax() bool {
+	return ss == maxNonStarters
+}
+
+// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
+const GraphemeJoiner = "\u034F"
+
+// reorderBuffer is used to normalize a single segment.  Characters inserted with
+// insert are decomposed and reordered based on CCC. The compose method can
+// be used to recombine characters.  Note that the byte buffer does not hold
+// the UTF-8 characters in order.  Only the rune array is maintained in sorted
+// order. flush writes the resulting segment to a byte array.
+type reorderBuffer struct {
+	rune  [maxBufferSize]Properties // Per character info.
+	byte  [maxByteBufferSize]byte   // UTF-8 buffer. Referenced by runeInfo.pos.
+	nbyte uint8                     // Number or bytes.
+	ss    streamSafe                // For limiting length of non-starter sequence.
+	nrune int                       // Number of runeInfos.
+	f     formInfo
+
+	src      input
+	nsrc     int
+	tmpBytes input
+
+	out    []byte
+	flushF func(*reorderBuffer) bool
+}
+
+func (rb *reorderBuffer) init(f Form, src []byte) {
+	rb.f = *formTable[f]
+	rb.src.setBytes(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) initString(f Form, src string) {
+	rb.f = *formTable[f]
+	rb.src.setString(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
+	rb.out = out
+	rb.flushF = f
+}
+
+// reset discards all characters from the buffer.
+func (rb *reorderBuffer) reset() {
+	rb.nrune = 0
+	rb.nbyte = 0
+}
+
+func (rb *reorderBuffer) doFlush() bool {
+	if rb.f.composing {
+		rb.compose()
+	}
+	res := rb.flushF(rb)
+	rb.reset()
+	return res
+}
+
+// appendFlush appends the normalized segment to rb.out.
+func appendFlush(rb *reorderBuffer) bool {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		rb.out = append(rb.out, rb.byte[start:end]...)
+	}
+	return true
+}
+
+// flush appends the normalized segment to out and resets rb.
+func (rb *reorderBuffer) flush(out []byte) []byte {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		out = append(out, rb.byte[start:end]...)
+	}
+	rb.reset()
+	return out
+}
+
+// flushCopy copies the normalized segment to buf and resets rb.
+// It returns the number of bytes written to buf.
+func (rb *reorderBuffer) flushCopy(buf []byte) int {
+	p := 0
+	for i := 0; i < rb.nrune; i++ {
+		runep := rb.rune[i]
+		p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
+	}
+	rb.reset()
+	return p
+}
+
+// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
+// It returns false if the buffer is not large enough to hold the rune.
+// It is used internally by insert and insertString only.
+func (rb *reorderBuffer) insertOrdered(info Properties) {
+	n := rb.nrune
+	b := rb.rune[:]
+	cc := info.ccc
+	if cc > 0 {
+		// Find insertion position + move elements to make room.
+		for ; n > 0; n-- {
+			if b[n-1].ccc <= cc {
+				break
+			}
+			b[n] = b[n-1]
+		}
+	}
+	rb.nrune += 1
+	pos := uint8(rb.nbyte)
+	rb.nbyte += utf8.UTFMax
+	info.pos = pos
+	b[n] = info
+}
+
+// insertErr is an error code returned by insert. Using this type instead
+// of error improves performance up to 20% for many of the benchmarks.
+type insertErr int
+
+const (
+	iSuccess insertErr = -iota
+	iShortDst
+	iShortSrc
+)
+
+// insertFlush inserts the given rune in the buffer ordered by CCC.
+// If a decomposition with multiple segments are encountered, they leading
+// ones are flushed.
+// It returns a non-zero error code if the rune was not inserted.
+func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+		return iSuccess
+	}
+	if info.hasDecomposition() {
+		return rb.insertDecomposed(info.Decomposition())
+	}
+	rb.insertSingle(src, i, info)
+	return iSuccess
+}
+
+// insertUnsafe inserts the given rune in the buffer ordered by CCC.
+// It is assumed there is sufficient space to hold the runes. It is the
+// responsibility of the caller to ensure this. This can be done by checking
+// the state returned by the streamSafe type.
+func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+	}
+	if info.hasDecomposition() {
+		// TODO: inline.
+		rb.insertDecomposed(info.Decomposition())
+	} else {
+		rb.insertSingle(src, i, info)
+	}
+}
+
+// insertDecomposed inserts an entry in to the reorderBuffer for each rune
+// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
+// It flushes the buffer on each new segment start.
+func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
+	rb.tmpBytes.setBytes(dcomp)
+	// As the streamSafe accounting already handles the counting for modifiers,
+	// we don't have to call next. However, we do need to keep the accounting
+	// intact when flushing the buffer.
+	for i := 0; i < len(dcomp); {
+		info := rb.f.info(rb.tmpBytes, i)
+		if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
+			return iShortDst
+		}
+		i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
+		rb.insertOrdered(info)
+	}
+	return iSuccess
+}
+
+// insertSingle inserts an entry in the reorderBuffer for the rune at
+// position i. info is the runeInfo for the rune at position i.
+func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
+	src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
+	rb.insertOrdered(info)
+}
+
+// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
+func (rb *reorderBuffer) insertCGJ() {
+	rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
+}
+
+// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
+func (rb *reorderBuffer) appendRune(r rune) {
+	bn := rb.nbyte
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.nbyte += utf8.UTFMax
+	rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
+	rb.nrune++
+}
+
+// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) assignRune(pos int, r rune) {
+	bn := rb.rune[pos].pos
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
+}
+
+// runeAt returns the rune at position n. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) runeAt(n int) rune {
+	inf := rb.rune[n]
+	r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
+	return r
+}
+
+// bytesAt returns the UTF-8 encoding of the rune at position n.
+// It is used for Hangul and recomposition.
+func (rb *reorderBuffer) bytesAt(n int) []byte {
+	inf := rb.rune[n]
+	return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
+}
+
+// For Hangul we combine algorithmically, instead of using tables.
+const (
+	hangulBase  = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
+	hangulBase0 = 0xEA
+	hangulBase1 = 0xB0
+	hangulBase2 = 0x80
+
+	hangulEnd  = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
+	hangulEnd0 = 0xED
+	hangulEnd1 = 0x9E
+	hangulEnd2 = 0xA4
+
+	jamoLBase  = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
+	jamoLBase0 = 0xE1
+	jamoLBase1 = 0x84
+	jamoLEnd   = 0x1113
+	jamoVBase  = 0x1161
+	jamoVEnd   = 0x1176
+	jamoTBase  = 0x11A7
+	jamoTEnd   = 0x11C3
+
+	jamoTCount   = 28
+	jamoVCount   = 21
+	jamoVTCount  = 21 * 28
+	jamoLVTCount = 19 * 21 * 28
+)
+
+const hangulUTF8Size = 3
+
+func isHangul(b []byte) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+func isHangulString(b string) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+// Caller must ensure len(b) >= 2.
+func isJamoVT(b []byte) bool {
+	// True if (rune & 0xff00) == jamoLBase
+	return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
+}
+
+func isHangulWithoutJamoT(b []byte) bool {
+	c, _ := utf8.DecodeRune(b)
+	c -= hangulBase
+	return c < jamoLVTCount && c%jamoTCount == 0
+}
+
+// decomposeHangul writes the decomposed Hangul to buf and returns the number
+// of bytes written.  len(buf) should be at least 9.
+func decomposeHangul(buf []byte, r rune) int {
+	const JamoUTF8Len = 3
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
+	utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
+	if x != 0 {
+		utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
+		return 3 * JamoUTF8Len
+	}
+	return 2 * JamoUTF8Len
+}
+
+// decomposeHangul algorithmically decomposes a Hangul rune into
+// its Jamo components.
+// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
+func (rb *reorderBuffer) decomposeHangul(r rune) {
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	rb.appendRune(jamoLBase + r/jamoVCount)
+	rb.appendRune(jamoVBase + r%jamoVCount)
+	if x != 0 {
+		rb.appendRune(jamoTBase + x)
+	}
+}
+
+// combineHangul algorithmically combines Jamo character components into Hangul.
+// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
+func (rb *reorderBuffer) combineHangul(s, i, k int) {
+	b := rb.rune[:]
+	bn := rb.nrune
+	for ; i < bn; i++ {
+		cccB := b[k-1].ccc
+		cccC := b[i].ccc
+		if cccB == 0 {
+			s = k - 1
+		}
+		if s != k-1 && cccB >= cccC {
+			// b[i] is blocked by greater-equal cccX below it
+			b[k] = b[i]
+			k++
+		} else {
+			l := rb.runeAt(s) // also used to compare to hangulBase
+			v := rb.runeAt(i) // also used to compare to jamoT
+			switch {
+			case jamoLBase <= l && l < jamoLEnd &&
+				jamoVBase <= v && v < jamoVEnd:
+				// 11xx plus 116x to LV
+				rb.assignRune(s, hangulBase+
+					(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
+			case hangulBase <= l && l < hangulEnd &&
+				jamoTBase < v && v < jamoTEnd &&
+				((l-hangulBase)%jamoTCount) == 0:
+				// ACxx plus 11Ax to LVT
+				rb.assignRune(s, l+v-jamoTBase)
+			default:
+				b[k] = b[i]
+				k++
+			}
+		}
+	}
+	rb.nrune = k
+}
+
+// compose recombines the runes in the buffer.
+// It should only be used to recompose a single segment, as it will not
+// handle alternations between Hangul and non-Hangul characters correctly.
+func (rb *reorderBuffer) compose() {
+	// UAX #15, section X5 , including Corrigendum #5
+	// "In any character sequence beginning with starter S, a character C is
+	//  blocked from S if and only if there is some character B between S
+	//  and C, and either B is a starter or it has the same or higher
+	//  combining class as C."
+	bn := rb.nrune
+	if bn == 0 {
+		return
+	}
+	k := 1
+	b := rb.rune[:]
+	for s, i := 0, 1; i < bn; i++ {
+		if isJamoVT(rb.bytesAt(i)) {
+			// Redo from start in Hangul mode. Necessary to support
+			// U+320E..U+321E in NFKC mode.
+			rb.combineHangul(s, i, k)
+			return
+		}
+		ii := b[i]
+		// We can only use combineForward as a filter if we later
+		// get the info for the combined character. This is more
+		// expensive than using the filter. Using combinesBackward()
+		// is safe.
+		if ii.combinesBackward() {
+			cccB := b[k-1].ccc
+			cccC := ii.ccc
+			blocked := false // b[i] blocked by starter or greater or equal CCC?
+			if cccB == 0 {
+				s = k - 1
+			} else {
+				blocked = s != k-1 && cccB >= cccC
+			}
+			if !blocked {
+				combined := combine(rb.runeAt(s), rb.runeAt(i))
+				if combined != 0 {
+					rb.assignRune(s, combined)
+					continue
+				}
+			}
+		}
+		b[k] = b[i]
+		k++
+	}
+	rb.nrune = k
+}

vendor/golang.org/x/text/unicode/norm/forminfo.go

@@ -0,0 +1,259 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+// This file contains Form-specific logic and wrappers for data in tables.go.
+
+// Rune info is stored in a separate trie per composing form. A composing form
+// and its corresponding decomposing form share the same trie.  Each trie maps
+// a rune to a uint16. The values take two forms.  For v >= 0x8000:
+//   bits
+//   15:    1 (inverse of NFD_QC bit of qcInfo)
+//   13..7: qcInfo (see below). isYesD is always true (no decompostion).
+//    6..0: ccc (compressed CCC value).
+// For v < 0x8000, the respective rune has a decomposition and v is an index
+// into a byte array of UTF-8 decomposition sequences and additional info and
+// has the form:
+//    <header> <decomp_byte>* [<tccc> [<lccc>]]
+// The header contains the number of bytes in the decomposition (excluding this
+// length byte). The two most significant bits of this length byte correspond
+// to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
+// The byte sequence is followed by a trailing and leading CCC if the values
+// for these are not zero.  The value of v determines which ccc are appended
+// to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
+// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
+// there is an additional leading ccc. The value of tccc itself is the
+// trailing CCC shifted left 2 bits. The two least-significant bits of tccc
+// are the number of trailing non-starters.
+
+const (
+	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
+	headerLenMask   = 0x3F // extract the length value from the header byte
+	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
+)
+
+// Properties provides access to normalization properties of a rune.
+type Properties struct {
+	pos   uint8  // start position in reorderBuffer; used in composition.go
+	size  uint8  // length of UTF-8 encoding of this rune
+	ccc   uint8  // leading canonical combining class (ccc if not decomposition)
+	tccc  uint8  // trailing canonical combining class (ccc if not decomposition)
+	nLead uint8  // number of leading non-starters.
+	flags qcInfo // quick check flags
+	index uint16
+}
+
+// functions dispatchable per form
+type lookupFunc func(b input, i int) Properties
+
+// formInfo holds Form-specific functions and tables.
+type formInfo struct {
+	form                     Form
+	composing, compatibility bool // form type
+	info                     lookupFunc
+	nextMain                 iterFunc
+}
+
+var formTable = []*formInfo{{
+	form:          NFC,
+	composing:     true,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFD,
+	composing:     false,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextDecomposed,
+}, {
+	form:          NFKC,
+	composing:     true,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFKD,
+	composing:     false,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextDecomposed,
+}}
+
+// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
+// unexpected behavior for the user.  For example, in NFD, there is a boundary
+// after 'a'.  However, 'a' might combine with modifiers, so from the application's
+// perspective it is not a good boundary. We will therefore always use the
+// boundaries for the combining variants.
+
+// BoundaryBefore returns true if this rune starts a new segment and
+// cannot combine with any rune on the left.
+func (p Properties) BoundaryBefore() bool {
+	if p.ccc == 0 && !p.combinesBackward() {
+		return true
+	}
+	// We assume that the CCC of the first character in a decomposition
+	// is always non-zero if different from info.ccc and that we can return
+	// false at this point. This is verified by maketables.
+	return false
+}
+
+// BoundaryAfter returns true if runes cannot combine with or otherwise
+// interact with this or previous runes.
+func (p Properties) BoundaryAfter() bool {
+	// TODO: loosen these conditions.
+	return p.isInert()
+}
+
+// We pack quick check data in 4 bits:
+//   5:    Combines forward  (0 == false, 1 == true)
+//   4..3: NFC_QC Yes(00), No (10), or Maybe (11)
+//   2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
+//   1..0: Number of trailing non-starters.
+//
+// When all 4 bits are zero, the character is inert, meaning it is never
+// influenced by normalization.
+type qcInfo uint8
+
+func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
+func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
+
+func (p Properties) combinesForward() bool  { return p.flags&0x20 != 0 }
+func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
+func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
+
+func (p Properties) isInert() bool {
+	return p.flags&qcInfoMask == 0 && p.ccc == 0
+}
+
+func (p Properties) multiSegment() bool {
+	return p.index >= firstMulti && p.index < endMulti
+}
+
+func (p Properties) nLeadingNonStarters() uint8 {
+	return p.nLead
+}
+
+func (p Properties) nTrailingNonStarters() uint8 {
+	return uint8(p.flags & 0x03)
+}
+
+// Decomposition returns the decomposition for the underlying rune
+// or nil if there is none.
+func (p Properties) Decomposition() []byte {
+	// TODO: create the decomposition for Hangul?
+	if p.index == 0 {
+		return nil
+	}
+	i := p.index
+	n := decomps[i] & headerLenMask
+	i++
+	return decomps[i : i+uint16(n)]
+}
+
+// Size returns the length of UTF-8 encoding of the rune.
+func (p Properties) Size() int {
+	return int(p.size)
+}
+
+// CCC returns the canonical combining class of the underlying rune.
+func (p Properties) CCC() uint8 {
+	if p.index >= firstCCCZeroExcept {
+		return 0
+	}
+	return ccc[p.ccc]
+}
+
+// LeadCCC returns the CCC of the first rune in the decomposition.
+// If there is no decomposition, LeadCCC equals CCC.
+func (p Properties) LeadCCC() uint8 {
+	return ccc[p.ccc]
+}
+
+// TrailCCC returns the CCC of the last rune in the decomposition.
+// If there is no decomposition, TrailCCC equals CCC.
+func (p Properties) TrailCCC() uint8 {
+	return ccc[p.tccc]
+}
+
+// Recomposition
+// We use 32-bit keys instead of 64-bit for the two codepoint keys.
+// This clips off the bits of three entries, but we know this will not
+// result in a collision. In the unlikely event that changes to
+// UnicodeData.txt introduce collisions, the compiler will catch it.
+// Note that the recomposition map for NFC and NFKC are identical.
+
+// combine returns the combined rune or 0 if it doesn't exist.
+func combine(a, b rune) rune {
+	key := uint32(uint16(a))<<16 + uint32(uint16(b))
+	return recompMap[key]
+}
+
+func lookupInfoNFC(b input, i int) Properties {
+	v, sz := b.charinfoNFC(i)
+	return compInfo(v, sz)
+}
+
+func lookupInfoNFKC(b input, i int) Properties {
+	v, sz := b.charinfoNFKC(i)
+	return compInfo(v, sz)
+}
+
+// Properties returns properties for the first rune in s.
+func (f Form) Properties(s []byte) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookup(s))
+	}
+	return compInfo(nfkcData.lookup(s))
+}
+
+// PropertiesString returns properties for the first rune in s.
+func (f Form) PropertiesString(s string) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookupString(s))
+	}
+	return compInfo(nfkcData.lookupString(s))
+}
+
+// compInfo converts the information contained in v and sz
+// to a Properties.  See the comment at the top of the file
+// for more information on the format.
+func compInfo(v uint16, sz int) Properties {
+	if v == 0 {
+		return Properties{size: uint8(sz)}
+	} else if v >= 0x8000 {
+		p := Properties{
+			size:  uint8(sz),
+			ccc:   uint8(v),
+			tccc:  uint8(v),
+			flags: qcInfo(v >> 8),
+		}
+		if p.ccc > 0 || p.combinesBackward() {
+			p.nLead = uint8(p.flags & 0x3)
+		}
+		return p
+	}
+	// has decomposition
+	h := decomps[v]
+	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
+	p := Properties{size: uint8(sz), flags: f, index: v}
+	if v >= firstCCC {
+		v += uint16(h&headerLenMask) + 1
+		c := decomps[v]
+		p.tccc = c >> 2
+		p.flags |= qcInfo(c & 0x3)
+		if v >= firstLeadingCCC {
+			p.nLead = c & 0x3
+			if v >= firstStarterWithNLead {
+				// We were tricked. Remove the decomposition.
+				p.flags &= 0x03
+				p.index = 0
+				return p
+			}
+			p.ccc = decomps[v+1]
+		}
+	}
+	return p
+}

vendor/golang.org/x/text/unicode/norm/input.go

@@ -0,0 +1,109 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+type input struct {
+	str   string
+	bytes []byte
+}
+
+func inputBytes(str []byte) input {
+	return input{bytes: str}
+}
+
+func inputString(str string) input {
+	return input{str: str}
+}
+
+func (in *input) setBytes(str []byte) {
+	in.str = ""
+	in.bytes = str
+}
+
+func (in *input) setString(str string) {
+	in.str = str
+	in.bytes = nil
+}
+
+func (in *input) _byte(p int) byte {
+	if in.bytes == nil {
+		return in.str[p]
+	}
+	return in.bytes[p]
+}
+
+func (in *input) skipASCII(p, max int) int {
+	if in.bytes == nil {
+		for ; p < max && in.str[p] < utf8.RuneSelf; p++ {
+		}
+	} else {
+		for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) skipContinuationBytes(p int) int {
+	if in.bytes == nil {
+		for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ {
+		}
+	} else {
+		for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) appendSlice(buf []byte, b, e int) []byte {
+	if in.bytes != nil {
+		return append(buf, in.bytes[b:e]...)
+	}
+	for i := b; i < e; i++ {
+		buf = append(buf, in.str[i])
+	}
+	return buf
+}
+
+func (in *input) copySlice(buf []byte, b, e int) int {
+	if in.bytes == nil {
+		return copy(buf, in.str[b:e])
+	}
+	return copy(buf, in.bytes[b:e])
+}
+
+func (in *input) charinfoNFC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfcData.lookupString(in.str[p:])
+	}
+	return nfcData.lookup(in.bytes[p:])
+}
+
+func (in *input) charinfoNFKC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfkcData.lookupString(in.str[p:])
+	}
+	return nfkcData.lookup(in.bytes[p:])
+}
+
+func (in *input) hangul(p int) (r rune) {
+	var size int
+	if in.bytes == nil {
+		if !isHangulString(in.str[p:]) {
+			return 0
+		}
+		r, size = utf8.DecodeRuneInString(in.str[p:])
+	} else {
+		if !isHangul(in.bytes[p:]) {
+			return 0
+		}
+		r, size = utf8.DecodeRune(in.bytes[p:])
+	}
+	if size != hangulUTF8Size {
+		return 0
+	}
+	return r
+}

vendor/golang.org/x/text/unicode/norm/iter.go

@@ -0,0 +1,457 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"fmt"
+	"unicode/utf8"
+)
+
+// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
+// sequence of starter and non-starter runes for the purpose of normalization.
+const MaxSegmentSize = maxByteBufferSize
+
+// An Iter iterates over a string or byte slice, while normalizing it
+// to a given Form.
+type Iter struct {
+	rb     reorderBuffer
+	buf    [maxByteBufferSize]byte
+	info   Properties // first character saved from previous iteration
+	next   iterFunc   // implementation of next depends on form
+	asciiF iterFunc
+
+	p        int    // current position in input source
+	multiSeg []byte // remainder of multi-segment decomposition
+}
+
+type iterFunc func(*Iter) []byte
+
+// Init initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) Init(f Form, src []byte) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.init(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIBytes
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+}
+
+// InitString initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) InitString(f Form, src string) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.initString(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIString
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+}
+
+// Seek sets the segment to be returned by the next call to Next to start
+// at position p.  It is the responsibility of the caller to set p to the
+// start of a segment.
+func (i *Iter) Seek(offset int64, whence int) (int64, error) {
+	var abs int64
+	switch whence {
+	case 0:
+		abs = offset
+	case 1:
+		abs = int64(i.p) + offset
+	case 2:
+		abs = int64(i.rb.nsrc) + offset
+	default:
+		return 0, fmt.Errorf("norm: invalid whence")
+	}
+	if abs < 0 {
+		return 0, fmt.Errorf("norm: negative position")
+	}
+	if int(abs) >= i.rb.nsrc {
+		i.setDone()
+		return int64(i.p), nil
+	}
+	i.p = int(abs)
+	i.multiSeg = nil
+	i.next = i.rb.f.nextMain
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+	return abs, nil
+}
+
+// returnSlice returns a slice of the underlying input type as a byte slice.
+// If the underlying is of type []byte, it will simply return a slice.
+// If the underlying is of type string, it will copy the slice to the buffer
+// and return that.
+func (i *Iter) returnSlice(a, b int) []byte {
+	if i.rb.src.bytes == nil {
+		return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
+	}
+	return i.rb.src.bytes[a:b]
+}
+
+// Pos returns the byte position at which the next call to Next will commence processing.
+func (i *Iter) Pos() int {
+	return i.p
+}
+
+func (i *Iter) setDone() {
+	i.next = nextDone
+	i.p = i.rb.nsrc
+}
+
+// Done returns true if there is no more input to process.
+func (i *Iter) Done() bool {
+	return i.p >= i.rb.nsrc
+}
+
+// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
+// For any input a and b for which f(a) == f(b), subsequent calls
+// to Next will return the same segments.
+// Modifying runes are grouped together with the preceding starter, if such a starter exists.
+// Although not guaranteed, n will typically be the smallest possible n.
+func (i *Iter) Next() []byte {
+	return i.next(i)
+}
+
+func nextASCIIBytes(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.setDone()
+		return i.rb.src.bytes[i.p:p]
+	}
+	if i.rb.src.bytes[p] < utf8.RuneSelf {
+		p0 := i.p
+		i.p = p
+		return i.rb.src.bytes[p0:p]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextASCIIString(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.setDone()
+		return i.buf[:1]
+	}
+	if i.rb.src.str[p] < utf8.RuneSelf {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.p = p
+		return i.buf[:1]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextHangul(i *Iter) []byte {
+	p := i.p
+	next := p + hangulUTF8Size
+	if next >= i.rb.nsrc {
+		i.setDone()
+	} else if i.rb.src.hangul(next) == 0 {
+		i.rb.ss.next(i.info)
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		i.next = i.rb.f.nextMain
+		return i.next(i)
+	}
+	i.p = next
+	return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
+}
+
+func nextDone(i *Iter) []byte {
+	return nil
+}
+
+// nextMulti is used for iterating over multi-segment decompositions
+// for decomposing normal forms.
+func nextMulti(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	// skip first rune
+	for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
+	}
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.multiSeg = d[j:]
+			return d[:j]
+		}
+		j += int(info.size)
+	}
+	// treat last segment as normal decomposition
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+// nextMultiNorm is used for iterating over multi-segment decompositions
+// for composing normal forms.
+func nextMultiNorm(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.rb.compose()
+			seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+			i.rb.insertUnsafe(input{bytes: d}, j, info)
+			i.multiSeg = d[j+int(info.size):]
+			return seg
+		}
+		i.rb.insertUnsafe(input{bytes: d}, j, info)
+		j += int(info.size)
+	}
+	i.multiSeg = nil
+	i.next = nextComposed
+	return doNormComposed(i)
+}
+
+// nextDecomposed is the implementation of Next for forms NFD and NFKD.
+func nextDecomposed(i *Iter) (next []byte) {
+	outp := 0
+	inCopyStart, outCopyStart := i.p, 0
+	for {
+		if sz := int(i.info.size); sz <= 1 {
+			i.rb.ss = 0
+			p := i.p
+			i.p++ // ASCII or illegal byte.  Either way, advance by 1.
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				return i.returnSlice(p, i.p)
+			} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+				i.next = i.asciiF
+				return i.returnSlice(p, i.p)
+			}
+			outp++
+		} else if d := i.info.Decomposition(); d != nil {
+			// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
+			// Case 1: there is a leftover to copy.  In this case the decomposition
+			// must begin with a modifier and should always be appended.
+			// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
+			p := outp + len(d)
+			if outp > 0 {
+				i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+				// TODO: this condition should not be possible, but we leave it
+				// in for defensive purposes.
+				if p > len(i.buf) {
+					return i.buf[:outp]
+				}
+			} else if i.info.multiSegment() {
+				// outp must be 0 as multi-segment decompositions always
+				// start a new segment.
+				if i.multiSeg == nil {
+					i.multiSeg = d
+					i.next = nextMulti
+					return nextMulti(i)
+				}
+				// We are in the last segment.  Treat as normal decomposition.
+				d = i.multiSeg
+				i.multiSeg = nil
+				p = len(d)
+			}
+			prevCC := i.info.tccc
+			if i.p += sz; i.p >= i.rb.nsrc {
+				i.setDone()
+				i.info = Properties{} // Force BoundaryBefore to succeed.
+			} else {
+				i.info = i.rb.f.info(i.rb.src, i.p)
+			}
+			switch i.rb.ss.next(i.info) {
+			case ssOverflow:
+				i.next = nextCGJDecompose
+				fallthrough
+			case ssStarter:
+				if outp > 0 {
+					copy(i.buf[outp:], d)
+					return i.buf[:p]
+				}
+				return d
+			}
+			copy(i.buf[outp:], d)
+			outp = p
+			inCopyStart, outCopyStart = i.p, outp
+			if i.info.ccc < prevCC {
+				goto doNorm
+			}
+			continue
+		} else if r := i.rb.src.hangul(i.p); r != 0 {
+			outp = decomposeHangul(i.buf[:], r)
+			i.p += hangulUTF8Size
+			inCopyStart, outCopyStart = i.p, outp
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				break
+			} else if i.rb.src.hangul(i.p) != 0 {
+				i.next = nextHangul
+				return i.buf[:outp]
+			}
+		} else {
+			p := outp + sz
+			if p > len(i.buf) {
+				break
+			}
+			outp = p
+			i.p += sz
+		}
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		prevCC := i.info.tccc
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := i.rb.ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	if outCopyStart == 0 {
+		return i.returnSlice(inCopyStart, i.p)
+	} else if inCopyStart < i.p {
+		i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	}
+	return i.buf[:outp]
+doNorm:
+	// Insert what we have decomposed so far in the reorderBuffer.
+	// As we will only reorder, there will always be enough room.
+	i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	i.rb.insertDecomposed(i.buf[0:outp])
+	return doNormDecomposed(i)
+}
+
+func doNormDecomposed(i *Iter) []byte {
+	for {
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if i.info.ccc == 0 {
+			break
+		}
+		if s := i.rb.ss.next(i.info); s == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+	}
+	// new segment or too many combining characters: exit normalization
+	return i.buf[:i.rb.flushCopy(i.buf[:])]
+}
+
+func nextCGJDecompose(i *Iter) []byte {
+	i.rb.ss = 0
+	i.rb.insertCGJ()
+	i.next = nextDecomposed
+	i.rb.ss.first(i.info)
+	buf := doNormDecomposed(i)
+	return buf
+}
+
+// nextComposed is the implementation of Next for forms NFC and NFKC.
+func nextComposed(i *Iter) []byte {
+	outp, startp := 0, i.p
+	var prevCC uint8
+	for {
+		if !i.info.isYesC() {
+			goto doNorm
+		}
+		prevCC = i.info.tccc
+		sz := int(i.info.size)
+		if sz == 0 {
+			sz = 1 // illegal rune: copy byte-by-byte
+		}
+		p := outp + sz
+		if p > len(i.buf) {
+			break
+		}
+		outp = p
+		i.p += sz
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+			i.rb.ss = 0
+			i.next = i.asciiF
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := i.rb.ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	return i.returnSlice(startp, i.p)
+doNorm:
+	// reset to start position
+	i.p = startp
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+	if i.info.multiSegment() {
+		d := i.info.Decomposition()
+		info := i.rb.f.info(input{bytes: d}, 0)
+		i.rb.insertUnsafe(input{bytes: d}, 0, info)
+		i.multiSeg = d[int(info.size):]
+		i.next = nextMultiNorm
+		return nextMultiNorm(i)
+	}
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}
+
+func doNormComposed(i *Iter) []byte {
+	// First rune should already be inserted.
+	for {
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if s := i.rb.ss.next(i.info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	}
+	i.rb.compose()
+	seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+	return seg
+}
+
+func nextCGJCompose(i *Iter) []byte {
+	i.rb.ss = 0 // instead of first
+	i.rb.insertCGJ()
+	i.next = nextComposed
+	// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
+	// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
+	// If we ever change that, insert a check here.
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}

vendor/golang.org/x/text/unicode/norm/normalize.go

@@ -0,0 +1,609 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Note: the file data_test.go that is generated should not be checked in.
+//go:generate go run maketables.go triegen.go
+//go:generate go test -tags test
+
+// Package norm contains types and functions for normalizing Unicode strings.
+package norm // import "golang.org/x/text/unicode/norm"
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// A Form denotes a canonical representation of Unicode code points.
+// The Unicode-defined normalization and equivalence forms are:
+//
+//   NFC   Unicode Normalization Form C
+//   NFD   Unicode Normalization Form D
+//   NFKC  Unicode Normalization Form KC
+//   NFKD  Unicode Normalization Form KD
+//
+// For a Form f, this documentation uses the notation f(x) to mean
+// the bytes or string x converted to the given form.
+// A position n in x is called a boundary if conversion to the form can
+// proceed independently on both sides:
+//   f(x) == append(f(x[0:n]), f(x[n:])...)
+//
+// References: http://unicode.org/reports/tr15/ and
+// http://unicode.org/notes/tn5/.
+type Form int
+
+const (
+	NFC Form = iota
+	NFD
+	NFKC
+	NFKD
+)
+
+// Bytes returns f(b). May return b if f(b) = b.
+func (f Form) Bytes(b []byte) []byte {
+	src := inputBytes(b)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return b
+	}
+	out := make([]byte, n, len(b))
+	copy(out, b[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b), out: out, flushF: appendFlush}
+	return doAppendInner(&rb, n)
+}
+
+// String returns f(s).
+func (f Form) String(s string) string {
+	src := inputString(s)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return s
+	}
+	out := make([]byte, n, len(s))
+	copy(out, s[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s), out: out, flushF: appendFlush}
+	return string(doAppendInner(&rb, n))
+}
+
+// IsNormal returns true if b == f(b).
+func (f Form) IsNormal(b []byte) bool {
+	src := inputBytes(b)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b)}
+	rb.setFlusher(nil, cmpNormalBytes)
+	for bp < len(b) {
+		rb.out = b[bp:]
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(b), true)
+	}
+	return true
+}
+
+func cmpNormalBytes(rb *reorderBuffer) bool {
+	b := rb.out
+	for i := 0; i < rb.nrune; i++ {
+		info := rb.rune[i]
+		if int(info.size) > len(b) {
+			return false
+		}
+		p := info.pos
+		pe := p + info.size
+		for ; p < pe; p++ {
+			if b[0] != rb.byte[p] {
+				return false
+			}
+			b = b[1:]
+		}
+	}
+	return true
+}
+
+// IsNormalString returns true if s == f(s).
+func (f Form) IsNormalString(s string) bool {
+	src := inputString(s)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s)}
+	rb.setFlusher(nil, func(rb *reorderBuffer) bool {
+		for i := 0; i < rb.nrune; i++ {
+			info := rb.rune[i]
+			if bp+int(info.size) > len(s) {
+				return false
+			}
+			p := info.pos
+			pe := p + info.size
+			for ; p < pe; p++ {
+				if s[bp] != rb.byte[p] {
+					return false
+				}
+				bp++
+			}
+		}
+		return true
+	})
+	for bp < len(s) {
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(s), true)
+	}
+	return true
+}
+
+// patchTail fixes a case where a rune may be incorrectly normalized
+// if it is followed by illegal continuation bytes. It returns the
+// patched buffer and whether the decomposition is still in progress.
+func patchTail(rb *reorderBuffer) bool {
+	info, p := lastRuneStart(&rb.f, rb.out)
+	if p == -1 || info.size == 0 {
+		return true
+	}
+	end := p + int(info.size)
+	extra := len(rb.out) - end
+	if extra > 0 {
+		// Potentially allocating memory. However, this only
+		// happens with ill-formed UTF-8.
+		x := make([]byte, 0)
+		x = append(x, rb.out[len(rb.out)-extra:]...)
+		rb.out = rb.out[:end]
+		decomposeToLastBoundary(rb)
+		rb.doFlush()
+		rb.out = append(rb.out, x...)
+		return false
+	}
+	buf := rb.out[p:]
+	rb.out = rb.out[:p]
+	decomposeToLastBoundary(rb)
+	if s := rb.ss.next(info); s == ssStarter {
+		rb.doFlush()
+		rb.ss.first(info)
+	} else if s == ssOverflow {
+		rb.doFlush()
+		rb.insertCGJ()
+		rb.ss = 0
+	}
+	rb.insertUnsafe(inputBytes(buf), 0, info)
+	return true
+}
+
+func appendQuick(rb *reorderBuffer, i int) int {
+	if rb.nsrc == i {
+		return i
+	}
+	end, _ := rb.f.quickSpan(rb.src, i, rb.nsrc, true)
+	rb.out = rb.src.appendSlice(rb.out, i, end)
+	return end
+}
+
+// Append returns f(append(out, b...)).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) Append(out []byte, src ...byte) []byte {
+	return f.doAppend(out, inputBytes(src), len(src))
+}
+
+func (f Form) doAppend(out []byte, src input, n int) []byte {
+	if n == 0 {
+		return out
+	}
+	ft := formTable[f]
+	// Attempt to do a quickSpan first so we can avoid initializing the reorderBuffer.
+	if len(out) == 0 {
+		p, _ := ft.quickSpan(src, 0, n, true)
+		out = src.appendSlice(out, 0, p)
+		if p == n {
+			return out
+		}
+		rb := reorderBuffer{f: *ft, src: src, nsrc: n, out: out, flushF: appendFlush}
+		return doAppendInner(&rb, p)
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: n}
+	return doAppend(&rb, out, 0)
+}
+
+func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
+	rb.setFlusher(out, appendFlush)
+	src, n := rb.src, rb.nsrc
+	doMerge := len(out) > 0
+	if q := src.skipContinuationBytes(p); q > p {
+		// Move leading non-starters to destination.
+		rb.out = src.appendSlice(rb.out, p, q)
+		p = q
+		doMerge = patchTail(rb)
+	}
+	fd := &rb.f
+	if doMerge {
+		var info Properties
+		if p < n {
+			info = fd.info(src, p)
+			if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
+				if p == 0 {
+					decomposeToLastBoundary(rb)
+				}
+				p = decomposeSegment(rb, p, true)
+			}
+		}
+		if info.size == 0 {
+			rb.doFlush()
+			// Append incomplete UTF-8 encoding.
+			return src.appendSlice(rb.out, p, n)
+		}
+		if rb.nrune > 0 {
+			return doAppendInner(rb, p)
+		}
+	}
+	p = appendQuick(rb, p)
+	return doAppendInner(rb, p)
+}
+
+func doAppendInner(rb *reorderBuffer, p int) []byte {
+	for n := rb.nsrc; p < n; {
+		p = decomposeSegment(rb, p, true)
+		p = appendQuick(rb, p)
+	}
+	return rb.out
+}
+
+// AppendString returns f(append(out, []byte(s))).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) AppendString(out []byte, src string) []byte {
+	return f.doAppend(out, inputString(src), len(src))
+}
+
+// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpan(b []byte) int {
+	n, _ := formTable[f].quickSpan(inputBytes(b), 0, len(b), true)
+	return n
+}
+
+// Span implements transform.SpanningTransformer. It returns a boundary n such
+// that b[0:n] == f(b[0:n]). It is not guaranteed to return the largest such n.
+func (f Form) Span(b []byte, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), atEOF)
+	if n < len(b) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// SpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) SpanString(s string, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputString(s), 0, len(s), atEOF)
+	if n < len(s) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and
+// whether any non-normalized parts were found. If atEOF is false, n will
+// not point past the last segment if this segment might be become
+// non-normalized by appending other runes.
+func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) {
+	var lastCC uint8
+	ss := streamSafe(0)
+	lastSegStart := i
+	for n = end; i < n; {
+		if j := src.skipASCII(i, n); i != j {
+			i = j
+			lastSegStart = i - 1
+			lastCC = 0
+			ss = 0
+			continue
+		}
+		info := f.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				// include incomplete runes
+				return n, true
+			}
+			return lastSegStart, true
+		}
+		// This block needs to be before the next, because it is possible to
+		// have an overflow for runes that are starters (e.g. with U+FF9E).
+		switch ss.next(info) {
+		case ssStarter:
+			lastSegStart = i
+		case ssOverflow:
+			return lastSegStart, false
+		case ssSuccess:
+			if lastCC > info.ccc {
+				return lastSegStart, false
+			}
+		}
+		if f.composing {
+			if !info.isYesC() {
+				break
+			}
+		} else {
+			if !info.isYesD() {
+				break
+			}
+		}
+		lastCC = info.ccc
+		i += int(info.size)
+	}
+	if i == n {
+		if !atEOF {
+			n = lastSegStart
+		}
+		return n, true
+	}
+	return lastSegStart, false
+}
+
+// QuickSpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpanString(s string) int {
+	n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true)
+	return n
+}
+
+// FirstBoundary returns the position i of the first boundary in b
+// or -1 if b contains no boundary.
+func (f Form) FirstBoundary(b []byte) int {
+	return f.firstBoundary(inputBytes(b), len(b))
+}
+
+func (f Form) firstBoundary(src input, nsrc int) int {
+	i := src.skipContinuationBytes(0)
+	if i >= nsrc {
+		return -1
+	}
+	fd := formTable[f]
+	ss := streamSafe(0)
+	// We should call ss.first here, but we can't as the first rune is
+	// skipped already. This means FirstBoundary can't really determine
+	// CGJ insertion points correctly. Luckily it doesn't have to.
+	for {
+		info := fd.info(src, i)
+		if info.size == 0 {
+			return -1
+		}
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+		i += int(info.size)
+		if i >= nsrc {
+			if !info.BoundaryAfter() && !ss.isMax() {
+				return -1
+			}
+			return nsrc
+		}
+	}
+}
+
+// FirstBoundaryInString returns the position i of the first boundary in s
+// or -1 if s contains no boundary.
+func (f Form) FirstBoundaryInString(s string) int {
+	return f.firstBoundary(inputString(s), len(s))
+}
+
+// NextBoundary reports the index of the boundary between the first and next
+// segment in b or -1 if atEOF is false and there are not enough bytes to
+// determine this boundary.
+func (f Form) NextBoundary(b []byte, atEOF bool) int {
+	return f.nextBoundary(inputBytes(b), len(b), atEOF)
+}
+
+// NextBoundaryInString reports the index of the boundary between the first and
+// next segment in b or -1 if atEOF is false and there are not enough bytes to
+// determine this boundary.
+func (f Form) NextBoundaryInString(s string, atEOF bool) int {
+	return f.nextBoundary(inputString(s), len(s), atEOF)
+}
+
+func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
+	if nsrc == 0 {
+		if atEOF {
+			return 0
+		}
+		return -1
+	}
+	fd := formTable[f]
+	info := fd.info(src, 0)
+	if info.size == 0 {
+		if atEOF {
+			return 1
+		}
+		return -1
+	}
+	ss := streamSafe(0)
+	ss.first(info)
+
+	for i := int(info.size); i < nsrc; i += int(info.size) {
+		info = fd.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				return i
+			}
+			return -1
+		}
+		// TODO: Using streamSafe to determine the boundary isn't the same as
+		// using BoundaryBefore. Determine which should be used.
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+	}
+	if !atEOF && !info.BoundaryAfter() && !ss.isMax() {
+		return -1
+	}
+	return nsrc
+}
+
+// LastBoundary returns the position i of the last boundary in b
+// or -1 if b contains no boundary.
+func (f Form) LastBoundary(b []byte) int {
+	return lastBoundary(formTable[f], b)
+}
+
+func lastBoundary(fd *formInfo, b []byte) int {
+	i := len(b)
+	info, p := lastRuneStart(fd, b)
+	if p == -1 {
+		return -1
+	}
+	if info.size == 0 { // ends with incomplete rune
+		if p == 0 { // starts with incomplete rune
+			return -1
+		}
+		i = p
+		info, p = lastRuneStart(fd, b[:i])
+		if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
+			return i
+		}
+	}
+	if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
+		return i
+	}
+	if info.BoundaryAfter() {
+		return i
+	}
+	ss := streamSafe(0)
+	v := ss.backwards(info)
+	for i = p; i >= 0 && v != ssStarter; i = p {
+		info, p = lastRuneStart(fd, b[:i])
+		if v = ss.backwards(info); v == ssOverflow {
+			break
+		}
+		if p+int(info.size) != i {
+			if p == -1 { // no boundary found
+				return -1
+			}
+			return i // boundary after an illegal UTF-8 encoding
+		}
+	}
+	return i
+}
+
+// decomposeSegment scans the first segment in src into rb. It inserts 0x034f
+// (Grapheme Joiner) when it encounters a sequence of more than 30 non-starters
+// and returns the number of bytes consumed from src or iShortDst or iShortSrc.
+func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
+	// Force one character to be consumed.
+	info := rb.f.info(rb.src, sp)
+	if info.size == 0 {
+		return 0
+	}
+	if s := rb.ss.next(info); s == ssStarter {
+		// TODO: this could be removed if we don't support merging.
+		if rb.nrune > 0 {
+			goto end
+		}
+	} else if s == ssOverflow {
+		rb.insertCGJ()
+		goto end
+	}
+	if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+		return int(err)
+	}
+	for {
+		sp += int(info.size)
+		if sp >= rb.nsrc {
+			if !atEOF && !info.BoundaryAfter() {
+				return int(iShortSrc)
+			}
+			break
+		}
+		info = rb.f.info(rb.src, sp)
+		if info.size == 0 {
+			if !atEOF {
+				return int(iShortSrc)
+			}
+			break
+		}
+		if s := rb.ss.next(info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			rb.insertCGJ()
+			break
+		}
+		if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+			return int(err)
+		}
+	}
+end:
+	if !rb.doFlush() {
+		return int(iShortDst)
+	}
+	return sp
+}
+
+// lastRuneStart returns the runeInfo and position of the last
+// rune in buf or the zero runeInfo and -1 if no rune was found.
+func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
+	p := len(buf) - 1
+	for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
+	}
+	if p < 0 {
+		return Properties{}, -1
+	}
+	return fd.info(inputBytes(buf), p), p
+}
+
+// decomposeToLastBoundary finds an open segment at the end of the buffer
+// and scans it into rb. Returns the buffer minus the last segment.
+func decomposeToLastBoundary(rb *reorderBuffer) {
+	fd := &rb.f
+	info, i := lastRuneStart(fd, rb.out)
+	if int(info.size) != len(rb.out)-i {
+		// illegal trailing continuation bytes
+		return
+	}
+	if info.BoundaryAfter() {
+		return
+	}
+	var add [maxNonStarters + 1]Properties // stores runeInfo in reverse order
+	padd := 0
+	ss := streamSafe(0)
+	p := len(rb.out)
+	for {
+		add[padd] = info
+		v := ss.backwards(info)
+		if v == ssOverflow {
+			// Note that if we have an overflow, it the string we are appending to
+			// is not correctly normalized. In this case the behavior is undefined.
+			break
+		}
+		padd++
+		p -= int(info.size)
+		if v == ssStarter || p < 0 {
+			break
+		}
+		info, i = lastRuneStart(fd, rb.out[:p])
+		if int(info.size) != p-i {
+			break
+		}
+	}
+	rb.ss = ss
+	// Copy bytes for insertion as we may need to overwrite rb.out.
+	var buf [maxBufferSize * utf8.UTFMax]byte
+	cp := buf[:copy(buf[:], rb.out[p:])]
+	rb.out = rb.out[:p]
+	for padd--; padd >= 0; padd-- {
+		info = add[padd]
+		rb.insertUnsafe(inputBytes(cp), 0, info)
+		cp = cp[info.size:]
+	}
+}

vendor/golang.org/x/text/unicode/norm/readwriter.go

@@ -0,0 +1,125 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "io"
+
+type normWriter struct {
+	rb  reorderBuffer
+	w   io.Writer
+	buf []byte
+}
+
+// Write implements the standard write interface.  If the last characters are
+// not at a normalization boundary, the bytes will be buffered for the next
+// write. The remaining bytes will be written on close.
+func (w *normWriter) Write(data []byte) (n int, err error) {
+	// Process data in pieces to keep w.buf size bounded.
+	const chunk = 4000
+
+	for len(data) > 0 {
+		// Normalize into w.buf.
+		m := len(data)
+		if m > chunk {
+			m = chunk
+		}
+		w.rb.src = inputBytes(data[:m])
+		w.rb.nsrc = m
+		w.buf = doAppend(&w.rb, w.buf, 0)
+		data = data[m:]
+		n += m
+
+		// Write out complete prefix, save remainder.
+		// Note that lastBoundary looks back at most 31 runes.
+		i := lastBoundary(&w.rb.f, w.buf)
+		if i == -1 {
+			i = 0
+		}
+		if i > 0 {
+			if _, err = w.w.Write(w.buf[:i]); err != nil {
+				break
+			}
+			bn := copy(w.buf, w.buf[i:])
+			w.buf = w.buf[:bn]
+		}
+	}
+	return n, err
+}
+
+// Close forces data that remains in the buffer to be written.
+func (w *normWriter) Close() error {
+	if len(w.buf) > 0 {
+		_, err := w.w.Write(w.buf)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Writer returns a new writer that implements Write(b)
+// by writing f(b) to w.  The returned writer may use an
+// an internal buffer to maintain state across Write calls.
+// Calling its Close method writes any buffered data to w.
+func (f Form) Writer(w io.Writer) io.WriteCloser {
+	wr := &normWriter{rb: reorderBuffer{}, w: w}
+	wr.rb.init(f, nil)
+	return wr
+}
+
+type normReader struct {
+	rb           reorderBuffer
+	r            io.Reader
+	inbuf        []byte
+	outbuf       []byte
+	bufStart     int
+	lastBoundary int
+	err          error
+}
+
+// Read implements the standard read interface.
+func (r *normReader) Read(p []byte) (int, error) {
+	for {
+		if r.lastBoundary-r.bufStart > 0 {
+			n := copy(p, r.outbuf[r.bufStart:r.lastBoundary])
+			r.bufStart += n
+			if r.lastBoundary-r.bufStart > 0 {
+				return n, nil
+			}
+			return n, r.err
+		}
+		if r.err != nil {
+			return 0, r.err
+		}
+		outn := copy(r.outbuf, r.outbuf[r.lastBoundary:])
+		r.outbuf = r.outbuf[0:outn]
+		r.bufStart = 0
+
+		n, err := r.r.Read(r.inbuf)
+		r.rb.src = inputBytes(r.inbuf[0:n])
+		r.rb.nsrc, r.err = n, err
+		if n > 0 {
+			r.outbuf = doAppend(&r.rb, r.outbuf, 0)
+		}
+		if err == io.EOF {
+			r.lastBoundary = len(r.outbuf)
+		} else {
+			r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf)
+			if r.lastBoundary == -1 {
+				r.lastBoundary = 0
+			}
+		}
+	}
+}
+
+// Reader returns a new reader that implements Read
+// by reading data from r and returning f(data).
+func (f Form) Reader(r io.Reader) io.Reader {
+	const chunk = 4000
+	buf := make([]byte, chunk)
+	rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf}
+	rr.rb.init(f, buf)
+	return rr
+}

vendor/golang.org/x/text/unicode/norm/transform.go

@@ -0,0 +1,88 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// Reset implements the Reset method of the transform.Transformer interface.
+func (Form) Reset() {}
+
+// Transform implements the Transform method of the transform.Transformer
+// interface. It may need to write segments of up to MaxSegmentSize at once.
+// Users should either catch ErrShortDst and allow dst to grow or have dst be at
+// least of size MaxTransformChunkSize to be guaranteed of progress.
+func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := 0
+	// Cap the maximum number of src bytes to check.
+	b := src
+	eof := atEOF
+	if ns := len(dst); ns < len(b) {
+		err = transform.ErrShortDst
+		eof = false
+		b = b[:ns]
+	}
+	i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof)
+	n += copy(dst[n:], b[n:i])
+	if !ok {
+		nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
+		return nDst + n, nSrc + n, err
+	}
+	if n < len(src) && !atEOF {
+		err = transform.ErrShortSrc
+	}
+	return n, n, err
+}
+
+func flushTransform(rb *reorderBuffer) bool {
+	// Write out (must fully fit in dst, or else it is an ErrShortDst).
+	if len(rb.out) < rb.nrune*utf8.UTFMax {
+		return false
+	}
+	rb.out = rb.out[rb.flushCopy(rb.out):]
+	return true
+}
+
+var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
+
+// transform implements the transform.Transformer interface. It is only called
+// when quickSpan does not pass for a given string.
+func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// TODO: get rid of reorderBuffer. See CL 23460044.
+	rb := reorderBuffer{}
+	rb.init(f, src)
+	for {
+		// Load segment into reorder buffer.
+		rb.setFlusher(dst[nDst:], flushTransform)
+		end := decomposeSegment(&rb, nSrc, atEOF)
+		if end < 0 {
+			return nDst, nSrc, errs[-end]
+		}
+		nDst = len(dst) - len(rb.out)
+		nSrc = end
+
+		// Next quickSpan.
+		end = rb.nsrc
+		eof := atEOF
+		if n := nSrc + len(dst) - nDst; n < end {
+			err = transform.ErrShortDst
+			end = n
+			eof = false
+		}
+		end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
+		n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
+		nSrc += n
+		nDst += n
+		if ok {
+			if n < rb.nsrc && !atEOF {
+				err = transform.ErrShortSrc
+			}
+			return nDst, nSrc, err
+		}
+	}
+}

vendor/golang.org/x/text/unicode/norm/trie.go

@@ -0,0 +1,54 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var nfcSparse = sparseBlocks{
+	values: nfcSparseValues[:],
+	offset: nfcSparseOffset[:],
+}
+
+var nfkcSparse = sparseBlocks{
+	values: nfkcSparseValues[:],
+	offset: nfkcSparseOffset[:],
+}
+
+var (
+	nfcData  = newNfcTrie(0)
+	nfkcData = newNfkcTrie(0)
+)
+
+// lookupValue determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

vendor/vendor.json

@@ -543,6 +543,12 @@
 			"revision": "b4690f45fa1cafc47b1c280c2e75116efe40cc13",
 			"revisionTime": "2017-02-15T08:41:58Z"
 		},
+		{
+			"checksumSHA1": "RcrB7tgYS/GMW4QrwVdMOTNqIU8=",
+			"path": "golang.org/x/net/idna",
+			"revision": "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec",
+			"revisionTime": "2018-01-12T01:53:59Z"
+		},
 		{
 			"checksumSHA1": "7EZyXN0EmZLgGxZxK01IJua4c8o=",
 			"path": "golang.org/x/net/websocket",
@@ -651,12 +657,30 @@
 			"revision": "85c29909967d7f171f821e7a42e7b7af76fb9598",
 			"revisionTime": "2017-02-11T12:01:23Z"
 		},
+		{
+			"checksumSHA1": "CbpjEkkOeh0fdM/V8xKDdI0AA88=",
+			"path": "golang.org/x/text/secure/bidirule",
+			"revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
+			"revisionTime": "2017-12-24T20:31:28Z"
+		},
 		{
 			"checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=",
 			"path": "golang.org/x/text/transform",
 			"revision": "85c29909967d7f171f821e7a42e7b7af76fb9598",
 			"revisionTime": "2017-02-11T12:01:23Z"
 		},
+		{
+			"checksumSHA1": "w8kDfZ1Ug+qAcVU0v8obbu3aDOY=",
+			"path": "golang.org/x/text/unicode/bidi",
+			"revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
+			"revisionTime": "2017-12-24T20:31:28Z"
+		},
+		{
+			"checksumSHA1": "BCNYmf4Ek93G4lk5x3ucNi/lTwA=",
+			"path": "golang.org/x/text/unicode/norm",
+			"revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
+			"revisionTime": "2017-12-24T20:31:28Z"
+		},
 		{
 			"checksumSHA1": "ikor+YKJu2eKwyFteBWhsb8IGy8=",
 			"path": "golang.org/x/tools/go/ast/astutil",

whisper/whisperv6/api.go

@@ -116,12 +116,17 @@ func (api *PublicWhisperAPI) SetMaxMessageSize(ctx context.Context, size uint32)
 	return true, api.w.SetMaxMessageSize(size)
 }
 
-// SetMinPow sets the minimum PoW for a message before it is accepted.
+// SetMinPow sets the minimum PoW, and notifies the peers.
 func (api *PublicWhisperAPI) SetMinPoW(ctx context.Context, pow float64) (bool, error) {
 	return true, api.w.SetMinimumPoW(pow)
 }
 
-// MarkTrustedPeer marks a peer trusted. , which will allow it to send historic (expired) messages.
+// SetBloomFilter sets the new value of bloom filter, and notifies the peers.
+func (api *PublicWhisperAPI) SetBloomFilter(ctx context.Context, bloom hexutil.Bytes) (bool, error) {
+	return true, api.w.SetBloomFilter(bloom)
+}
+
+// MarkTrustedPeer marks a peer trusted, which will allow it to send historic (expired) messages.
 // Note: This function is not adding new nodes, the node needs to exists as a peer.
 func (api *PublicWhisperAPI) MarkTrustedPeer(ctx context.Context, enode string) (bool, error) {
 	n, err := discover.ParseNode(enode)

whisper/whisperv6/doc.go

@@ -35,7 +35,6 @@ import (
 )
 
 const (
-	EnvelopeVersion    = uint64(0)
 	ProtocolVersion    = uint64(6)
 	ProtocolVersionStr = "6.0"
 	ProtocolName       = "shh"
@@ -52,11 +51,14 @@ const (
 	paddingMask   = byte(3)
 	signatureFlag = byte(4)
 
-	TopicLength     = 4
-	signatureLength = 65
-	aesKeyLength    = 32
-	AESNonceLength  = 12
-	keyIdSize       = 32
+	TopicLength     = 4  // in bytes
+	signatureLength = 65 // in bytes
+	aesKeyLength    = 32 // in bytes
+	AESNonceLength  = 12 // in bytes
+	keyIdSize       = 32 // in bytes
+	bloomFilterSize = 64 // in bytes
+
+	EnvelopeHeaderLength = 20
 
 	MaxMessageSize        = uint32(10 * 1024 * 1024) // maximum accepted size of a message.
 	DefaultMaxMessageSize = uint32(1024 * 1024)
@@ -68,10 +70,8 @@ const (
 	expirationCycle   = time.Second
 	transmissionCycle = 300 * time.Millisecond
 
-	DefaultTTL     = 50 // seconds
-	SynchAllowance = 10 // seconds
-
-	EnvelopeHeaderLength = 20
+	DefaultTTL           = 50 // seconds
+	DefaultSyncAllowance = 10 // seconds
 )
 
 type unknownVersionError uint64

whisper/whisperv6/envelope.go

@@ -42,9 +42,11 @@ type Envelope struct {
 	Data   []byte
 	Nonce  uint64
 
-	pow  float64     // Message-specific PoW as described in the Whisper specification.
-	hash common.Hash // Cached hash of the envelope to avoid rehashing every time.
-	// Don't access hash directly, use Hash() function instead.
+	pow float64 // Message-specific PoW as described in the Whisper specification.
+
+	// the following variables should not be accessed directly, use the corresponding function instead: Hash(), Bloom()
+	hash  common.Hash // Cached hash of the envelope to avoid rehashing every time.
+	bloom []byte
 }
 
 // size returns the size of envelope as it is sent (i.e. public fields only)
@@ -227,3 +229,30 @@ func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) {
 	}
 	return msg
 }
+
+// Bloom maps 4-bytes Topic into 64-byte bloom filter with 3 bits set (at most).
+func (e *Envelope) Bloom() []byte {
+	if e.bloom == nil {
+		e.bloom = TopicToBloom(e.Topic)
+	}
+	return e.bloom
+}
+
+// TopicToBloom converts the topic (4 bytes) to the bloom filter (64 bytes)
+func TopicToBloom(topic TopicType) []byte {
+	b := make([]byte, bloomFilterSize)
+	var index [3]int
+	for j := 0; j < 3; j++ {
+		index[j] = int(topic[j])
+		if (topic[3] & (1 << uint(j))) != 0 {
+			index[j] += 256
+		}
+	}
+
+	for j := 0; j < 3; j++ {
+		byteIndex := index[j] / 8
+		bitIndex := index[j] % 8
+		b[byteIndex] = (1 << uint(bitIndex))
+	}
+	return b
+}

whisper/whisperv6/peer.go

@@ -36,6 +36,7 @@ type Peer struct {
 
 	trusted        bool
 	powRequirement float64
+	bloomFilter    []byte // may contain nil in case of full node
 
 	known *set.Set // Messages already known by the peer to avoid wasting bandwidth
 
@@ -74,8 +75,12 @@ func (p *Peer) handshake() error {
 	// Send the handshake status message asynchronously
 	errc := make(chan error, 1)
 	go func() {
-		errc <- p2p.Send(p.ws, statusCode, ProtocolVersion)
+		pow := p.host.MinPow()
+		powConverted := math.Float64bits(pow)
+		bloom := p.host.BloomFilter()
+		errc <- p2p.SendItems(p.ws, statusCode, ProtocolVersion, powConverted, bloom)
 	}()
+
 	// Fetch the remote status packet and verify protocol match
 	packet, err := p.ws.ReadMsg()
 	if err != nil {
@@ -85,14 +90,42 @@ func (p *Peer) handshake() error {
 		return fmt.Errorf("peer [%x] sent packet %x before status packet", p.ID(), packet.Code)
 	}
 	s := rlp.NewStream(packet.Payload, uint64(packet.Size))
-	peerVersion, err := s.Uint()
+	_, err = s.List()
 	if err != nil {
 		return fmt.Errorf("peer [%x] sent bad status message: %v", p.ID(), err)
 	}
+	peerVersion, err := s.Uint()
+	if err != nil {
+		return fmt.Errorf("peer [%x] sent bad status message (unable to decode version): %v", p.ID(), err)
+	}
 	if peerVersion != ProtocolVersion {
 		return fmt.Errorf("peer [%x]: protocol version mismatch %d != %d", p.ID(), peerVersion, ProtocolVersion)
 	}
-	// Wait until out own status is consumed too
+
+	// only version is mandatory, subsequent parameters are optional
+	powRaw, err := s.Uint()
+	if err == nil {
+		pow := math.Float64frombits(powRaw)
+		if math.IsInf(pow, 0) || math.IsNaN(pow) || pow < 0.0 {
+			return fmt.Errorf("peer [%x] sent bad status message: invalid pow", p.ID())
+		}
+		p.powRequirement = pow
+
+		var bloom []byte
+		err = s.Decode(&bloom)
+		if err == nil {
+			sz := len(bloom)
+			if sz != bloomFilterSize && sz != 0 {
+				return fmt.Errorf("peer [%x] sent bad status message: wrong bloom filter size %d", p.ID(), sz)
+			}
+			if isFullNode(bloom) {
+				p.bloomFilter = nil
+			} else {
+				p.bloomFilter = bloom
+			}
+		}
+	}
+
 	if err := <-errc; err != nil {
 		return fmt.Errorf("peer [%x] failed to send status packet: %v", p.ID(), err)
 	}
@@ -156,7 +189,7 @@ func (p *Peer) broadcast() error {
 	envelopes := p.host.Envelopes()
 	bundle := make([]*Envelope, 0, len(envelopes))
 	for _, envelope := range envelopes {
-		if !p.marked(envelope) && envelope.PoW() >= p.powRequirement {
+		if !p.marked(envelope) && envelope.PoW() >= p.powRequirement && p.bloomMatch(envelope) {
 			bundle = append(bundle, envelope)
 		}
 	}
@@ -186,3 +219,16 @@ func (p *Peer) notifyAboutPowRequirementChange(pow float64) error {
 	i := math.Float64bits(pow)
 	return p2p.Send(p.ws, powRequirementCode, i)
 }
+
+func (p *Peer) notifyAboutBloomFilterChange(bloom []byte) error {
+	return p2p.Send(p.ws, bloomFilterExCode, bloom)
+}
+
+func (p *Peer) bloomMatch(env *Envelope) bool {
+	if p.bloomFilter == nil {
+		// no filter - full node, accepts all envelops
+		return true
+	}
+
+	return bloomFilterMatch(p.bloomFilter, env.Bloom())
+}

whisper/whisperv6/peer_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/ecdsa"
 	"fmt"
+	mrand "math/rand"
 	"net"
 	"sync"
 	"testing"
@@ -87,6 +88,9 @@ var nodes [NumNodes]*TestNode
 var sharedKey []byte = []byte("some arbitrary data here")
 var sharedTopic TopicType = TopicType{0xF, 0x1, 0x2, 0}
 var expectedMessage []byte = []byte("per rectum ad astra")
+var masterBloomFilter []byte
+var masterPow = 0.00000001
+var round int = 1
 
 func TestSimulation(t *testing.T) {
 	// create a chain of whisper nodes,
@@ -104,8 +108,13 @@ func TestSimulation(t *testing.T) {
 	// check if each node have received and decrypted exactly one message
 	checkPropagation(t, true)
 
-	// send protocol-level messages (powRequirementCode) and check the new PoW requirement values
-	powReqExchange(t)
+	// check if Status message was correctly decoded
+	checkBloomFilterExchange(t)
+	checkPowExchange(t)
+
+	// send new pow and bloom exchange messages
+	resetParams(t)
+	round++
 
 	// node #1 sends one expected (decryptable) message
 	sendMsg(t, true, 1)
@@ -113,18 +122,65 @@ func TestSimulation(t *testing.T) {
 	// check if each node (except node #0) have received and decrypted exactly one message
 	checkPropagation(t, false)
 
+	for i := 1; i < NumNodes; i++ {
+		time.Sleep(20 * time.Millisecond)
+		sendMsg(t, true, i)
+	}
+
+	// check if corresponding protocol-level messages were correctly decoded
+	checkPowExchangeForNodeZero(t)
+	checkBloomFilterExchange(t)
+
 	stopServers()
 }
 
+func resetParams(t *testing.T) {
+	// change pow only for node zero
+	masterPow = 7777777.0
+	nodes[0].shh.SetMinimumPoW(masterPow)
+
+	// change bloom for all nodes
+	masterBloomFilter = TopicToBloom(sharedTopic)
+	for i := 0; i < NumNodes; i++ {
+		nodes[i].shh.SetBloomFilter(masterBloomFilter)
+	}
+}
+
+func initBloom(t *testing.T) {
+	masterBloomFilter = make([]byte, bloomFilterSize)
+	_, err := mrand.Read(masterBloomFilter)
+	if err != nil {
+		t.Fatalf("rand failed: %s.", err)
+	}
+
+	msgBloom := TopicToBloom(sharedTopic)
+	masterBloomFilter = addBloom(masterBloomFilter, msgBloom)
+	for i := 0; i < 32; i++ {
+		masterBloomFilter[i] = 0xFF
+	}
+
+	if !bloomFilterMatch(masterBloomFilter, msgBloom) {
+		t.Fatalf("bloom mismatch on initBloom.")
+	}
+}
+
 func initialize(t *testing.T) {
+	initBloom(t)
+
 	var err error
 	ip := net.IPv4(127, 0, 0, 1)
 	port0 := 30303
 
 	for i := 0; i < NumNodes; i++ {
 		var node TestNode
+		b := make([]byte, bloomFilterSize)
+		copy(b, masterBloomFilter)
 		node.shh = New(&DefaultConfig)
-		node.shh.SetMinimumPowTest(0.00000001)
+		node.shh.SetMinimumPoW(masterPow)
+		node.shh.SetBloomFilter(b)
+		if !bytes.Equal(node.shh.BloomFilter(), masterBloomFilter) {
+			t.Fatalf("bloom mismatch on init.")
+		}
 		node.shh.Start(nil)
 		topics := make([]TopicType, 0)
 		topics = append(topics, sharedTopic)
@@ -206,7 +262,7 @@ func checkPropagation(t *testing.T, includingNodeZero bool) {
 		for i := first; i < NumNodes; i++ {
 			f := nodes[i].shh.GetFilter(nodes[i].filerId)
 			if f == nil {
-				t.Fatalf("failed to get filterId %s from node %d.", nodes[i].filerId, i)
+				t.Fatalf("failed to get filterId %s from node %d, round %d.", nodes[i].filerId, i, round)
 			}
 
 			mail := f.Retrieve()
@@ -332,34 +388,43 @@ func TestPeerBasic(t *testing.T) {
 	}
 }
 
-func powReqExchange(t *testing.T) {
-	for i, node := range nodes {
-		for peer := range node.shh.peers {
-			if peer.powRequirement > 1000.0 {
-				t.Fatalf("node %d: one of the peers' pow requirement is too big (%f).", i, peer.powRequirement)
-			}
-		}
-	}
-
-	const pow float64 = 7777777.0
-	nodes[0].shh.SetMinimumPoW(pow)
-
-	// wait until all the messages are delivered
-	time.Sleep(64 * time.Millisecond)
-
+func checkPowExchangeForNodeZero(t *testing.T) {
 	cnt := 0
 	for i, node := range nodes {
 		for peer := range node.shh.peers {
 			if peer.peer.ID() == discover.PubkeyID(&nodes[0].id.PublicKey) {
 				cnt++
-				if peer.powRequirement != pow {
+				if peer.powRequirement != masterPow {
 					t.Fatalf("node %d: failed to set the new pow requirement.", i)
 				}
 			}
 		}
 	}
-
 	if cnt == 0 {
 		t.Fatalf("no matching peers found.")
 	}
 }
+
+func checkPowExchange(t *testing.T) {
+	for i, node := range nodes {
+		for peer := range node.shh.peers {
+			if peer.peer.ID() != discover.PubkeyID(&nodes[0].id.PublicKey) {
+				if peer.powRequirement != masterPow {
+					t.Fatalf("node %d: failed to exchange pow requirement in round %d; expected %f, got %f",
+						i, round, masterPow, peer.powRequirement)
+				}
+			}
+		}
+	}
+}
+
+func checkBloomFilterExchange(t *testing.T) {
+	for i, node := range nodes {
+		for peer := range node.shh.peers {
+			if !bytes.Equal(peer.bloomFilter, masterBloomFilter) {
+				t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got",
+					i, round, masterBloomFilter, peer.bloomFilter)
+			}
+		}
+	}
+}

whisper/whisperv6/whisper.go

@@ -48,9 +48,12 @@ type Statistics struct {
 }
 
 const (
-	minPowIdx     = iota // Minimal PoW required by the whisper node
-	maxMsgSizeIdx = iota // Maximal message length allowed by the whisper node
-	overflowIdx   = iota // Indicator of message queue overflow
+	maxMsgSizeIdx           = iota // Maximal message length allowed by the whisper node
+	overflowIdx                    // Indicator of message queue overflow
+	minPowIdx                      // Minimal PoW required by the whisper node
+	minPowToleranceIdx             // Minimal PoW tolerated by the whisper node for a limited time
+	bloomFilterIdx                 // Bloom filter for topics of interest for this node
+	bloomFilterToleranceIdx        // Bloom filter tolerated by the whisper node for a limited time
 )
 
 // Whisper represents a dark communication interface through the Ethereum
@@ -76,7 +79,7 @@ type Whisper struct {
 
 	settings syncmap.Map // holds configuration settings that can be dynamically changed
 
-	reactionAllowance int // maximum time in seconds allowed to process the whisper-related messages
+	syncAllowance int // maximum time in seconds allowed to process the whisper-related messages
 
 	statsMu sync.Mutex // guard stats
 	stats   Statistics // Statistics of whisper node
@@ -91,15 +94,15 @@ func New(cfg *Config) *Whisper {
 	}
 
 	whisper := &Whisper{
-		privateKeys:       make(map[string]*ecdsa.PrivateKey),
-		symKeys:           make(map[string][]byte),
-		envelopes:         make(map[common.Hash]*Envelope),
-		expirations:       make(map[uint32]*set.SetNonTS),
-		peers:             make(map[*Peer]struct{}),
-		messageQueue:      make(chan *Envelope, messageQueueLimit),
-		p2pMsgQueue:       make(chan *Envelope, messageQueueLimit),
-		quit:              make(chan struct{}),
-		reactionAllowance: SynchAllowance,
+		privateKeys:   make(map[string]*ecdsa.PrivateKey),
+		symKeys:       make(map[string][]byte),
+		envelopes:     make(map[common.Hash]*Envelope),
+		expirations:   make(map[uint32]*set.SetNonTS),
+		peers:         make(map[*Peer]struct{}),
+		messageQueue:  make(chan *Envelope, messageQueueLimit),
+		p2pMsgQueue:   make(chan *Envelope, messageQueueLimit),
+		quit:          make(chan struct{}),
+		syncAllowance: DefaultSyncAllowance,
 	}
 
 	whisper.filters = NewFilters(whisper)
@@ -126,11 +129,55 @@ func New(cfg *Config) *Whisper {
 	return whisper
 }
 
+// MinPow returns the PoW value required by this node.
 func (w *Whisper) MinPow() float64 {
-	val, _ := w.settings.Load(minPowIdx)
+	val, exist := w.settings.Load(minPowIdx)
+	if !exist || val == nil {
+		return DefaultMinimumPoW
+	}
+	v, ok := val.(float64)
+	if !ok {
+		log.Error("Error loading minPowIdx, using default")
+		return DefaultMinimumPoW
+	}
+	return v
+}
+
+// MinPowTolerance returns the value of minimum PoW which is tolerated for a limited
+// time after PoW was changed. If sufficient time have elapsed or no change of PoW
+// have ever occurred, the return value will be the same as return value of MinPow().
+func (w *Whisper) MinPowTolerance() float64 {
+	val, exist := w.settings.Load(minPowToleranceIdx)
+	if !exist || val == nil {
+		return DefaultMinimumPoW
+	}
 	return val.(float64)
 }
 
+// BloomFilter returns the aggregated bloom filter for all the topics of interest.
+// The nodes are required to send only messages that match the advertised bloom filter.
+// If a message does not match the bloom, it will tantamount to spam, and the peer will
+// be disconnected.
+func (w *Whisper) BloomFilter() []byte {
+	val, exist := w.settings.Load(bloomFilterIdx)
+	if !exist || val == nil {
+		return nil
+	}
+	return val.([]byte)
+}
+
+// BloomFilterTolerance returns the bloom filter which is tolerated for a limited
+// time after new bloom was advertised to the peers. If sufficient time have elapsed
+// or no change of bloom filter have ever occurred, the return value will be the same
+// as return value of BloomFilter().
+func (w *Whisper) BloomFilterTolerance() []byte {
+	val, exist := w.settings.Load(bloomFilterToleranceIdx)
+	if !exist || val == nil {
+		return nil
+	}
+	return val.([]byte)
+}
+
 // MaxMessageSize returns the maximum accepted message size.
 func (w *Whisper) MaxMessageSize() uint32 {
 	val, _ := w.settings.Load(maxMsgSizeIdx)
@@ -180,18 +227,40 @@ func (w *Whisper) SetMaxMessageSize(size uint32) error {
 	return nil
 }
 
+// SetBloomFilter sets the new bloom filter
+func (w *Whisper) SetBloomFilter(bloom []byte) error {
+	if len(bloom) != bloomFilterSize {
+		return fmt.Errorf("invalid bloom filter size: %d", len(bloom))
+	}
+
+	b := make([]byte, bloomFilterSize)
+	copy(b, bloom)
+
+	w.settings.Store(bloomFilterIdx, b)
+	w.notifyPeersAboutBloomFilterChange(b)
+
+	go func() {
+		// allow some time before all the peers have processed the notification
+		time.Sleep(time.Duration(w.syncAllowance) * time.Second)
+		w.settings.Store(bloomFilterToleranceIdx, b)
+	}()
+
+	return nil
+}
+
 // SetMinimumPoW sets the minimal PoW required by this node
 func (w *Whisper) SetMinimumPoW(val float64) error {
 	if val < 0.0 {
 		return fmt.Errorf("invalid PoW: %f", val)
 	}
 
+	w.settings.Store(minPowIdx, val)
 	w.notifyPeersAboutPowRequirementChange(val)
 
 	go func() {
 		// allow some time before all the peers have processed the notification
-		time.Sleep(time.Duration(w.reactionAllowance) * time.Second)
-		w.settings.Store(minPowIdx, val)
+		time.Sleep(time.Duration(w.syncAllowance) * time.Second)
+		w.settings.Store(minPowToleranceIdx, val)
 	}()
 
 	return nil
@@ -199,21 +268,13 @@ func (w *Whisper) SetMinimumPoW(val float64) error {
 
 // SetMinimumPoW sets the minimal PoW in test environment
 func (w *Whisper) SetMinimumPowTest(val float64) {
-	w.notifyPeersAboutPowRequirementChange(val)
 	w.settings.Store(minPowIdx, val)
+	w.notifyPeersAboutPowRequirementChange(val)
+	w.settings.Store(minPowToleranceIdx, val)
 }
 
 func (w *Whisper) notifyPeersAboutPowRequirementChange(pow float64) {
-	arr := make([]*Peer, len(w.peers))
-	i := 0
-
-	w.peerMu.Lock()
-	for p := range w.peers {
-		arr[i] = p
-		i++
-	}
-	w.peerMu.Unlock()
-
+	arr := w.getPeers()
 	for _, p := range arr {
 		err := p.notifyAboutPowRequirementChange(pow)
 		if err != nil {
@@ -221,11 +282,37 @@ func (w *Whisper) notifyPeersAboutPowRequirementChange(pow float64) {
 			err = p.notifyAboutPowRequirementChange(pow)
 		}
 		if err != nil {
-			log.Warn("oversized message received", "peer", p.ID(), "error", err)
+			log.Warn("failed to notify peer about new pow requirement", "peer", p.ID(), "error", err)
 		}
 	}
 }
 
+func (w *Whisper) notifyPeersAboutBloomFilterChange(bloom []byte) {
+	arr := w.getPeers()
+	for _, p := range arr {
+		err := p.notifyAboutBloomFilterChange(bloom)
+		if err != nil {
+			// allow one retry
+			err = p.notifyAboutBloomFilterChange(bloom)
+		}
+		if err != nil {
+			log.Warn("failed to notify peer about new bloom filter", "peer", p.ID(), "error", err)
+		}
+	}
+}
+
+func (w *Whisper) getPeers() []*Peer {
+	arr := make([]*Peer, len(w.peers))
+	i := 0
+	w.peerMu.Lock()
+	for p := range w.peers {
+		arr[i] = p
+		i++
+	}
+	w.peerMu.Unlock()
+	return arr
+}
+
 // getPeer retrieves peer by ID
 func (w *Whisper) getPeer(peerID []byte) (*Peer, error) {
 	w.peerMu.Lock()
@@ -459,7 +546,28 @@ func (w *Whisper) GetSymKey(id string) ([]byte, error) {
 // Subscribe installs a new message handler used for filtering, decrypting
 // and subsequent storing of incoming messages.
 func (w *Whisper) Subscribe(f *Filter) (string, error) {
-	return w.filters.Install(f)
+	s, err := w.filters.Install(f)
+	if err == nil {
+		w.updateBloomFilter(f)
+	}
+	return s, err
+}
+
+// updateBloomFilter recalculates the new value of bloom filter,
+// and informs the peers if necessary.
+func (w *Whisper) updateBloomFilter(f *Filter) {
+	aggregate := make([]byte, bloomFilterSize)
+	for _, t := range f.Topics {
+		top := BytesToTopic(t)
+		b := TopicToBloom(top)
+		aggregate = addBloom(aggregate, b)
+	}
+
+	if !bloomFilterMatch(w.BloomFilter(), aggregate) {
+		// existing bloom filter must be updated
+		aggregate = addBloom(w.BloomFilter(), aggregate)
+		w.SetBloomFilter(aggregate)
+	}
 }
 
 // GetFilter returns the filter by id.
@@ -592,7 +700,21 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
 			}
 			p.powRequirement = f
 		case bloomFilterExCode:
-			// to be implemented
+			var bloom []byte
+			err := packet.Decode(&bloom)
+			if err == nil && len(bloom) != bloomFilterSize {
+				err = fmt.Errorf("wrong bloom filter size %d", len(bloom))
+			}
+
+			if err != nil {
+				log.Warn("failed to decode bloom filter exchange message, peer will be disconnected", "peer", p.peer.ID(), "err", err)
+				return errors.New("invalid bloom filter exchange message")
+			}
+			if isFullNode(bloom) {
+				p.bloomFilter = nil
+			} else {
+				p.bloomFilter = bloom
+			}
 		case p2pMessageCode:
 			// peer-to-peer message, sent directly to peer bypassing PoW checks, etc.
 			// this message is not supposed to be forwarded to other peers, and
@@ -633,7 +755,7 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) {
 	sent := envelope.Expiry - envelope.TTL
 
 	if sent > now {
-		if sent-SynchAllowance > now {
+		if sent-DefaultSyncAllowance > now {
 			return false, fmt.Errorf("envelope created in the future [%x]", envelope.Hash())
 		} else {
 			// recalculate PoW, adjusted for the time difference, plus one second for latency
@@ -642,7 +764,7 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) {
 	}
 
 	if envelope.Expiry < now {
-		if envelope.Expiry+SynchAllowance*2 < now {
+		if envelope.Expiry+DefaultSyncAllowance*2 < now {
 			return false, fmt.Errorf("very old message")
 		} else {
 			log.Debug("expired envelope dropped", "hash", envelope.Hash().Hex())
@@ -655,11 +777,22 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) {
 	}
 
 	if envelope.PoW() < wh.MinPow() {
-		log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex())
-		return false, nil // drop envelope without error for now
+		// maybe the value was recently changed, and the peers did not adjust yet.
+		// in this case the previous value is retrieved by MinPowTolerance()
+		// for a short period of peer synchronization.
+		if envelope.PoW() < wh.MinPowTolerance() {
+			return false, fmt.Errorf("envelope with low PoW received: PoW=%f, hash=[%v]", envelope.PoW(), envelope.Hash().Hex())
+		}
+	}
 
-		// once the status message includes the PoW requirement, an error should be returned here:
-		//return false, fmt.Errorf("envelope with low PoW dropped: PoW=%f, hash=[%v]", envelope.PoW(), envelope.Hash().Hex())
+	if !bloomFilterMatch(wh.BloomFilter(), envelope.Bloom()) {
+		// maybe the value was recently changed, and the peers did not adjust yet.
+		// in this case the previous value is retrieved by BloomFilterTolerance()
+		// for a short period of peer synchronization.
+		if !bloomFilterMatch(wh.BloomFilterTolerance(), envelope.Bloom()) {
+			return false, fmt.Errorf("envelope does not match bloom filter, hash=[%v], bloom: \n%x \n%x \n%x",
+				envelope.Hash().Hex(), wh.BloomFilter(), envelope.Bloom(), envelope.Topic)
+		}
 	}
 
 	hash := envelope.Hash()
@@ -897,3 +1030,40 @@ func GenerateRandomID() (id string, err error) {
 	id = common.Bytes2Hex(buf)
 	return id, err
 }
+
+func isFullNode(bloom []byte) bool {
+	if bloom == nil {
+		return true
+	}
+	for _, b := range bloom {
+		if b != 255 {
+			return false
+		}
+	}
+	return true
+}
+
+func bloomFilterMatch(filter, sample []byte) bool {
+	if filter == nil {
+		// full node, accepts all messages
+		return true
+	}
+
+	for i := 0; i < bloomFilterSize; i++ {
+		f := filter[i]
+		s := sample[i]
+		if (f | s) != f {
+			return false
+		}
+	}
+
+	return true
+}
+
+func addBloom(a, b []byte) []byte {
+	c := make([]byte, bloomFilterSize)
+	for i := 0; i < bloomFilterSize; i++ {
+		c[i] = a[i] | b[i]
+	}
+	return c
+}

whisper/whisperv6/whisper_test.go

@@ -843,3 +843,64 @@ func TestSymmetricSendKeyMismatch(t *testing.T) {
 		t.Fatalf("received a message when keys weren't matching")
 	}
 }
+
+func TestBloom(t *testing.T) {
+	topic := TopicType{0, 0, 255, 6}
+	b := TopicToBloom(topic)
+	x := make([]byte, bloomFilterSize)
+	x[0] = byte(1)
+	x[32] = byte(1)
+	x[bloomFilterSize-1] = byte(128)
+	if !bloomFilterMatch(x, b) || !bloomFilterMatch(b, x) {
+		t.Fatalf("bloom filter does not match the mask")
+	}
+
+	_, err := mrand.Read(b)
+	if err != nil {
+		t.Fatalf("math rand error")
+	}
+	_, err = mrand.Read(x)
+	if err != nil {
+		t.Fatalf("math rand error")
+	}
+	if !bloomFilterMatch(b, b) {
+		t.Fatalf("bloom filter does not match self")
+	}
+	x = addBloom(x, b)
+	if !bloomFilterMatch(x, b) {
+		t.Fatalf("bloom filter does not match combined bloom")
+	}
+	if !isFullNode(nil) {
+		t.Fatalf("isFullNode did not recognize nil as full node")
+	}
+	x[17] = 254
+	if isFullNode(x) {
+		t.Fatalf("isFullNode false positive")
+	}
+	for i := 0; i < bloomFilterSize; i++ {
+		b[i] = byte(255)
+	}
+	if !isFullNode(b) {
+		t.Fatalf("isFullNode false negative")
+	}
+	if bloomFilterMatch(x, b) {
+		t.Fatalf("bloomFilterMatch false positive")
+	}
+	if !bloomFilterMatch(b, x) {
+		t.Fatalf("bloomFilterMatch false negative")
+	}
+
+	w := New(&DefaultConfig)
+	f := w.BloomFilter()
+	if f != nil {
+		t.Fatalf("wrong bloom on creation")
+	}
+	err = w.SetBloomFilter(x)
+	if err != nil {
+		t.Fatalf("failed to set bloom filter: %s", err)
+	}
+	f = w.BloomFilter()
+	if !bloomFilterMatch(f, x) || !bloomFilterMatch(x, f) {
+		t.Fatalf("retireved wrong bloom filter")
+	}
+}