forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 06:36:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of https://github.com/ethereum/go-ethereum

Browse source
This commit is contained in:
Zahoor Mohamed 2016-12-21 20:33:51 +05:30
commit 954e67b5b8
77 changed files with 3403 additions and 1141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.travis.yml
View file

@ -13,15 +13,15 @@ matrix:
go: 1.6.2
- os: linux
dist: trusty
go: 1.7
go: 1.7.4
- os: osx
go: 1.7
go: 1.7.4
# This builder does the Ubuntu PPA and Linux Azure uploads
- os: linux
dist: trusty
sudo: required
go: 1.7
go: 1.7.4
env:
- ubuntu-ppa
- azure-linux
@ -55,7 +55,7 @@ matrix:
# This builder does the OSX Azure, Android Maven and Azure and iOS CocoaPods and Azure uploads
- os: osx
go: 1.7
go: 1.7.4
env:
- azure-osx
- mobile

4
README.md
View file

@ -39,9 +39,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow insolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
| `bzzd` | swarm daemon. This is the entrypoint for the swarm network. `bzzd --help` for command line options. See https://swarm-guide.readthedocs.io for swarm documentation. |
| `bzzup` | swarm command line file uploader. `bzzup --help` for command line options |
| `bzzhash` | command to calculate the swarm hash of a file or directory. `bzzhash --help` for command line options |
| `swarm` | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. |
## Running geth

2
VERSION
View file

@ -1 +1 @@
1.5.5
1.5.6

14
accounts/abi/type.go
View file

@ -33,7 +33,7 @@ const (
FixedBytesTy
BytesTy
HashTy
RealTy
FixedpointTy
)
// Type is the reflection of the supported argument type
@ -57,16 +57,16 @@ var (
// Types can be in the format of:
//
// Input = Type [ "[" [ Number ] "]" ] Name .
// Type = [ "u" ] "int" [ Number ] .
// Type = [ "u" ] "int" [ Number ] [ x ] [ Number ].
//
// Examples:
//
// string int uint real
// string int uint fixed
// string32 int8 uint8 uint[]
// address int256 uint256 real[2]
fullTypeRegex = regexp.MustCompile("([a-zA-Z0-9]+)(\\[([0-9]*)?\\])?")
// address int256 uint256 fixed128x128[2]
fullTypeRegex = regexp.MustCompile("([a-zA-Z0-9]+)(\\[([0-9]*)\\])?")
// typeRegex parses the abi sub types
typeRegex = regexp.MustCompile("([a-zA-Z]+)([0-9]*)?")
typeRegex = regexp.MustCompile("([a-zA-Z]+)(([0-9]+)(x([0-9]+))?)?")
)
// NewType creates a new reflection type of abi type given in t.
@ -97,7 +97,7 @@ func NewType(t string) (typ Type, err error) {
parsedType := typeRegex.FindAllStringSubmatch(res[1], -1)[0]
// varSize is the size of the variable
var varSize int
if len(parsedType[2]) > 0 {
if len(parsedType[3]) > 0 {
var err error
varSize, err = strconv.Atoi(parsedType[2])
if err != nil {

78
accounts/abi/type_test.go Normal file
View file

@ -0,0 +1,78 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package abi
import (
"reflect"
"testing"
)
// typeWithoutStringer is a alias for the Type type which simply doesn't implement
// the stringer interface to allow printing type details in the tests below.
type typeWithoutStringer Type
// Tests that all allowed types get recognized by the type parser.
func TestTypeRegexp(t *testing.T) {
tests := []struct {
blob string
kind Type
}{
{"int", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
{"int8", Type{Kind: reflect.Int8, Type: big_t, Size: 8, T: IntTy, stringKind: "int8"}},
{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
{"int[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
{"int[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
{"int32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
{"int32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
{"uint", Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}},
{"uint8", Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}},
{"uint256", Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}},
{"uint[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
{"uint[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
{"uint32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint32, Type: big_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
{"uint32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{Kind: reflect.Uint32, Type: big_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
{"bytes", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}},
{"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}},
{"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
{"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}},
{"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
{"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
{"string", Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}},
{"string[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
{"string[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}, stringKind: "string[2]"}},
{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
{"address[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
{"address[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
// TODO when fixed types are implemented properly
// {"fixed", Type{}},
// {"fixed128x128", Type{}},
// {"fixed[]", Type{}},
// {"fixed[2]", Type{}},
// {"fixed128x128[]", Type{}},
// {"fixed128x128[2]", Type{}},
}
for i, tt := range tests {
typ, err := NewType(tt.blob)
if err != nil {
t.Errorf("type %d: failed to parse type string: %v", i, err)
}
if !reflect.DeepEqual(typ, tt.kind) {
t.Errorf("type %d: parsed type mismatch:\n have %+v\n want %+v", i, typeWithoutStringer(typ), typeWithoutStringer(tt.kind))
}
}
}

4
appveyor.yml
View file

@ -22,8 +22,8 @@ environment:
install:
- rmdir C:\go /s /q
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.3.windows-%GETH_ARCH%.zip
- 7z x go1.7.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-%GETH_ARCH%.zip
- 7z x go1.7.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version
- gcc --version

20
build/ci.go
View file

@ -72,9 +72,7 @@ var (
executablePath("abigen"),
executablePath("evm"),
executablePath("geth"),
executablePath("bzzd"),
executablePath("bzzhash"),
executablePath("bzzup"),
executablePath("swarm"),
executablePath("rlpdump"),
}
@ -93,16 +91,8 @@ var (
Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
},
{
Name: "bzzd",
Description: "Ethereum Swarm daemon",
},
{
Name: "bzzup",
Description: "Ethereum Swarm command line file/directory uploader",
},
{
Name: "bzzhash",
Description: "Ethereum Swarm file/directory hash calculator",
Name: "swarm",
Description: "Ethereum Swarm daemon and tools",
},
{
Name: "abigen",
@ -112,7 +102,8 @@ var (
// Distros for which packages are created.
// Note: vivid is unsupported because there is no golang-1.6 package for it.
debDistros = []string{"trusty", "wily", "xenial", "yakkety"}
// Note: wily is unsupported because it was officially deprecated on lanchpad.
debDistros = []string{"trusty", "xenial", "yakkety"}
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@ -638,6 +629,7 @@ func doWindowsInstaller(cmdline []string) {
build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil)
build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData)
build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools)
build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil)
build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil)
build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755)
build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755)

5
build/nsis.geth.nsi
View file

@ -17,8 +17,12 @@
#
# Requirements:
# - NSIS, http://nsis.sourceforge.net/Main_Page
# - NSIS Large Strings build, http://nsis.sourceforge.net/Special_Builds
# - SFP, http://nsis.sourceforge.net/NSIS_Simple_Firewall_Plugin (put dll in NSIS\Plugins\x86-ansi)
#
# After intalling NSIS extra the NSIS Large Strings build zip and replace the makensis.exe and the
# files found in Stub.
#
# based on: http://nsis.sourceforge.net/A_simple_installer_with_start_menu_shortcut_and_uninstaller
#
# TODO:
@ -37,6 +41,7 @@ RequestExecutionLevel admin
SetCompressor /SOLID lzma
!include LogicLib.nsh
!include PathUpdate.nsh
!include EnvVarUpdate.nsh
!macro VerifyUserIsAdmin

5
build/nsis.install.nsh
View file

@ -37,8 +37,9 @@ Section "Geth" GETH_IDX
${EnvVarUpdate} $0 "ETHEREUM_SOCKET" "R" "HKLM" "\\.\pipe\geth.ipc"
${EnvVarUpdate} $0 "ETHEREUM_SOCKET" "A" "HKLM" "\\.\pipe\geth.ipc"
# Add geth to PATH
${EnvVarUpdate} $0 "PATH" "A" "HKLM" $INSTDIR
# Add instdir to PATH
Push "$INSTDIR"
Call AddToPath
SectionEnd
# Install optional develop tools.

153
build/nsis.pathupdate.nsh Normal file
View file

@ -0,0 +1,153 @@
!include "WinMessages.nsh"
; see https://support.microsoft.com/en-us/kb/104011
!define Environ 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"'
; HKEY_LOCAL_MACHINE = 0x80000002
; AddToPath - Appends dir to PATH
; (does not work on Win9x/ME)
;
; Usage:
; Push "dir"
; Call AddToPath
Function AddToPath
Exch $0
Push $1
Push $2
Push $3
Push $4
; NSIS ReadRegStr returns empty string on string overflow
; Native calls are used here to check actual length of PATH
; $4 = RegOpenKey(HKEY_LOCAL_MACHINE, "SYSTEM\CurrentControlSet\Control\Session Manager\Environment", &$3)
System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4"
IntCmp $4 0 0 done done
; $4 = RegQueryValueEx($3, "PATH", (DWORD*)0, (DWORD*)0, &$1, ($2=NSIS_MAX_STRLEN, &$2))
; RegCloseKey($3)
System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4"
System::Call "advapi32::RegCloseKey(i $3)"
IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA
DetailPrint "AddToPath: original length $2 > ${NSIS_MAX_STRLEN}"
MessageBox MB_OK "PATH not updated, original length $2 > ${NSIS_MAX_STRLEN}"
Goto done
IntCmp $4 0 +5 ; $4 != NO_ERROR
IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND
DetailPrint "AddToPath: unexpected error code $4"
Goto done
StrCpy $1 ""
; Check if already in PATH
Push "$1;"
Push "$0;"
Call StrStr
Pop $2
StrCmp $2 "" 0 done
Push "$1;"
Push "$0\;"
Call StrStr
Pop $2
StrCmp $2 "" 0 done
; Prevent NSIS string overflow
StrLen $2 $0
StrLen $3 $1
IntOp $2 $2 + $3
IntOp $2 $2 + 2 ; $2 = strlen(dir) + strlen(PATH) + sizeof(";")
IntCmp $2 ${NSIS_MAX_STRLEN} +4 +4 0
DetailPrint "AddToPath: new length $2 > ${NSIS_MAX_STRLEN}"
MessageBox MB_OK "PATH not updated, new length $2 > ${NSIS_MAX_STRLEN}."
Goto done
; Append dir to PATH
DetailPrint "Add to PATH: $0"
StrCpy $2 $1 1 -1
StrCmp $2 ";" 0 +2
StrCpy $1 $1 -1 ; remove trailing ';'
StrCmp $1 "" +2 ; no leading ';'
StrCpy $0 "$1;$0"
WriteRegExpandStr ${Environ} "PATH" $0
SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000
done:
Pop $4
Pop $3
Pop $2
Pop $1
Pop $0
FunctionEnd
; RemoveFromPath - Removes dir from PATH
;
; Usage:
; Push "dir"
; Call RemoveFromPath
Function un.RemoveFromPath
Exch $0
Push $1
Push $2
Push $3
Push $4
Push $5
Push $6
; NSIS ReadRegStr returns empty string on string overflow
; Native calls are used here to check actual length of PATH
; $4 = RegOpenKey(HKEY_LOCAL_MACHINE, "SYSTEM\CurrentControlSet\Control\Session Manager\Environment", &$3)
System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4"
IntCmp $4 0 0 done done
; $4 = RegQueryValueEx($3, "PATH", (DWORD*)0, (DWORD*)0, &$1, ($2=NSIS_MAX_STRLEN, &$2))
; RegCloseKey($3)
System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4"
System::Call "advapi32::RegCloseKey(i $3)"
IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA
DetailPrint "RemoveFromPath: original length $2 > ${NSIS_MAX_STRLEN}"
MessageBox MB_OK "PATH not updated, original length $2 > ${NSIS_MAX_STRLEN}"
Goto done
IntCmp $4 0 +5 ; $4 != NO_ERROR
IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND
DetailPrint "RemoveFromPath: unexpected error code $4"
Goto done
StrCpy $1 ""
; length < ${NSIS_MAX_STRLEN} -> ReadRegStr can be used
ReadRegStr $1 ${Environ} "PATH"
StrCpy $5 $1 1 -1
StrCmp $5 ";" +2
StrCpy $1 "$1;" ; ensure trailing ';'
Push $1
Push "$0;"
Call un.StrStr
Pop $2 ; pos of our dir
StrCmp $2 "" done
DetailPrint "Remove from PATH: $0"
StrLen $3 "$0;"
StrLen $4 $2
StrCpy $5 $1 -$4 ; $5 is now the part before the path to remove
StrCpy $6 $2 "" $3 ; $6 is now the part after the path to remove
StrCpy $3 "$5$6"
StrCpy $5 $3 1 -1
StrCmp $5 ";" 0 +2
StrCpy $3 $3 -1 ; remove trailing ';'
WriteRegExpandStr ${Environ} "PATH" $3
SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000
done:
Pop $6
Pop $5
Pop $4
Pop $3
Pop $2
Pop $1
Pop $0
FunctionEnd

3
build/nsis.uninstall.nsh
View file

@ -25,7 +25,8 @@ Section "Uninstall"
${un.EnvVarUpdate} $0 "ETHEREUM_SOCKET" "R" "HKLM" "\\.\pipe\geth.ipc"
# Remove install directory from PATH
${un.EnvVarUpdate} $0 "PATH" "R" "HKLM" $INSTDIR
Push "$INSTDIR"
Call un.RemoveFromPath
# Cleanup registry (deletes all sub keys)
DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\${GROUPNAME} ${APPNAME}"

24
cmd/geth/library.c
View file

@ -1,24 +0,0 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Simple wrapper to translate the API exposed methods and types to inthernal
// Go versions of the same types.
#include "_cgo_export.h"
int run(const char* args) {
return doRun((char*)args);
}

46
cmd/geth/library.go
View file

@ -1,46 +0,0 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Contains a simple library definition to allow creating a Geth instance from
// straight C code.
package main
// #ifdef __cplusplus
// extern "C" {
// #endif
//
// extern int run(const char*);
//
// #ifdef __cplusplus
// }
// #endif
import "C"
import (
"fmt"
"os"
"strings"
)
//export doRun
func doRun(args *C.char) C.int {
// This is equivalent to geth.main, just modified to handle the function arg passing
if err := app.Run(strings.Split("geth "+C.GoString(args), " ")); err != nil {
fmt.Fprintln(os.Stderr, err)
return -1
}
return 0
}

56
cmd/geth/library_android.go
View file

@ -1,56 +0,0 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Contains specialized code for running Geth on Android.
package main
// #include <android/log.h>
// #cgo LDFLAGS: -llog
import "C"
import (
"bufio"
"os"
)
func init() {
// Redirect the standard output and error to logcat
oldStdout, oldStderr := os.Stdout, os.Stderr
outRead, outWrite, _ := os.Pipe()
errRead, errWrite, _ := os.Pipe()
os.Stdout = outWrite
os.Stderr = errWrite
go func() {
scanner := bufio.NewScanner(outRead)
for scanner.Scan() {
line := scanner.Text()
C.__android_log_write(C.ANDROID_LOG_INFO, C.CString("Stdout"), C.CString(line))
oldStdout.WriteString(line + "\n")
}
}()
go func() {
scanner := bufio.NewScanner(errRead)
for scanner.Scan() {
line := scanner.Text()
C.__android_log_write(C.ANDROID_LOG_INFO, C.CString("Stderr"), C.CString(line))
oldStderr.WriteString(line + "\n")
}
}()
}

19
cmd/bzzhash/main.go → cmd/swarm/hash.go
View file

@ -19,22 +19,21 @@ package main
import (
"fmt"
"log"
"os"
"runtime"
"github.com/ethereum/go-ethereum/swarm/storage"
"gopkg.in/urfave/cli.v1"
)
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
if len(os.Args) < 2 {
fmt.Println("Usage: bzzhash <file name>")
os.Exit(0)
func hash(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
log.Fatal("Usage: swarm hash <file name>")
}
f, err := os.Open(os.Args[1])
f, err := os.Open(args[0])
if err != nil {
fmt.Println("Error opening file " + os.Args[1])
fmt.Println("Error opening file " + args[1])
os.Exit(1)
}
@ -42,7 +41,7 @@ func main() {
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
log.Fatalf("%v\n", err)
} else {
fmt.Printf("%v\n", key)
}

105
cmd/bzzd/main.go → cmd/swarm/main.go
View file

@ -43,11 +43,21 @@ import (
"gopkg.in/urfave/cli.v1"
)
const clientIdentifier = "bzzd"
const (
clientIdentifier = "swarm"
versionString = "0.2"
)
var (
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
app = utils.NewApp(gitCommit, "Ethereum Swarm server daemon")
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
app = utils.NewApp(gitCommit, "Ethereum Swarm")
testbetBootNodes = []string{
"enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429",
"enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430",
"enode://fe29b82319b734ce1ec68b84657d57145fee237387e63273989d354486731e59f78858e452ef800a020559da22dcca759536e6aa5517c53930d29ce0b1029286@13.74.157.139:30431",
"enode://1d7187e7bde45cf0bee489ce9852dd6d1a0d9aa67a33a6b8e6db8a4fbc6fcfa6f0f1a5419343671521b863b187d1c73bad3603bae66421d157ffef357669ddb8@13.74.157.139:30432",
"enode://0e4cba800f7b1ee73673afa6a4acead4018f0149d2e3216be3f133318fd165b324cd71b81fbe1e80deac8dbf56e57a49db7be67f8b9bc81bd2b7ee496434fb5d@13.74.157.139:30433",
}
)
var (
@ -65,7 +75,7 @@ var (
}
SwarmNetworkIdFlag = cli.IntFlag{
Name: "bzznetworkid",
Usage: "Network identifier (integer, default 322=swarm testnet)",
Usage: "Network identifier (integer, default 3=swarm testnet)",
Value: network.NetworkId,
}
SwarmConfigPathFlag = cli.StringFlag{
@ -85,10 +95,25 @@ var (
Usage: "URL of the Ethereum API provider",
Value: node.DefaultIPCEndpoint("geth"),
}
SwarmApiFlag = cli.StringFlag{
Name: "bzzapi",
Usage: "Swarm HTTP endpoint",
Value: "http://127.0.0.1:8500",
}
SwarmRecursiveUploadFlag = cli.BoolFlag{
Name: "recursive",
Usage: "Upload directories recursively",
}
SwarmWantManifestFlag = cli.BoolTFlag{
Name: "manifest",
Usage: "Automatic manifest upload",
}
SwarmUploadDefaultPath = cli.StringFlag{
Name: "defaultpath",
Usage: "path to file served for empty url path (none)",
}
)
var defaultBootnodes = []string{}
func init() {
// Override flag defaults so bzzd can run alongside geth.
utils.ListenPortFlag.Value = 30399
@ -96,8 +121,39 @@ func init() {
utils.IPCApiFlag.Value = "admin, bzz, chequebook, debug, rpc, web3"
// Set up the cli app.
app.Commands = nil
app.Action = bzzd
app.HideVersion = true // we have a command to print the version
app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
app.Commands = []cli.Command{
cli.Command{
Action: version,
Name: "version",
Usage: "Print version numbers",
ArgsUsage: " ",
Description: `
The output of this command is supposed to be machine-readable.
`,
},
cli.Command{
Action: upload,
Name: "up",
Usage: "upload a file or directory to swarm using the HTTP API",
ArgsUsage: " <file>",
Description: `
"upload a file or directory to swarm using the HTTP API and prints the root hash",
`,
},
cli.Command{
Action: hash,
Name: "hash",
Usage: "print the swarm hash of a file or directory",
ArgsUsage: " <file>",
Description: `
Prints the swarm hash of file or directory.
`,
},
}
app.Flags = []cli.Flag{
utils.IdentityFlag,
utils.DataDirFlag,
@ -123,6 +179,11 @@ func init() {
SwarmAccountFlag,
SwarmNetworkIdFlag,
ChequebookAddrFlag,
// upload flags
SwarmApiFlag,
SwarmRecursiveUploadFlag,
SwarmWantManifestFlag,
SwarmUploadDefaultPath,
}
app.Flags = append(app.Flags, debug.Flags...)
app.Before = func(ctx *cli.Context) error {
@ -142,17 +203,33 @@ func main() {
}
}
func version(ctx *cli.Context) error {
fmt.Println(strings.Title(clientIdentifier))
fmt.Println("Version:", versionString)
if gitCommit != "" {
fmt.Println("Git Commit:", gitCommit)
}
fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
fmt.Println("Go Version:", runtime.Version())
fmt.Println("OS:", runtime.GOOS)
fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
return nil
}
func bzzd(ctx *cli.Context) error {
stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
registerBzzService(ctx, stack)
utils.StartNode(stack)
networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name)
// Add bootnodes as initial peers.
if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
bootnodes := strings.Split(ctx.GlobalString(utils.BootnodesFlag.Name), ",")
injectBootnodes(stack.Server(), bootnodes)
} else {
injectBootnodes(stack.Server(), defaultBootnodes)
if networkId == 3 {
injectBootnodes(stack.Server(), testbetBootNodes)
}
}
stack.Wait()
@ -182,13 +259,11 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
boot := func(ctx *node.ServiceContext) (node.Service, error) {
var client *ethclient.Client
if ethapi == "" {
err = fmt.Errorf("use ethapi flag to connect to a an eth client and talk to the blockchain")
} else {
if len(ethapi) > 0 {
client, err = ethclient.Dial(ethapi)
}
if err != nil {
utils.Fatalf("Can't connect: %v", err)
if err != nil {
utils.Fatalf("Can't connect: %v", err)
}
}
return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled)
}

112
cmd/bzzup/main.go → cmd/swarm/upload.go
View file

@ -20,7 +20,6 @@ package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
@ -28,58 +27,83 @@ import (
"mime"
"net/http"
"os"
"os/user"
"path"
"path/filepath"
"strings"
"gopkg.in/urfave/cli.v1"
)
func main() {
func upload(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapiFlag = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint")
recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively")
manifestFlag = flag.Bool("manifest", true, "Skip automatic manifest upload")
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
recursive = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name)
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name)
)
log.SetOutput(os.Stderr)
log.SetFlags(0)
flag.Parse()
if flag.NArg() != 1 {
if len(args) != 1 {
log.Fatal("need filename as the first and only argument")
}
var (
file = flag.Arg(0)
client = &client{api: strings.TrimSuffix(*bzzapiFlag, "/")}
file = args[0]
client = &client{api: bzzapi}
mroot manifest
entry manifestEntry
)
fi, err := os.Stat(file)
fi, err := os.Stat(expandPath(file))
if err != nil {
log.Fatal(err)
}
if fi.IsDir() {
if !*recursiveFlag {
if !recursive {
log.Fatal("argument is a directory and recursive upload is disabled")
}
mroot, err = client.uploadDirectory(file)
mroot, err = client.uploadDirectory(file, defaultPath)
} else {
mroot, err = client.uploadFile(file, fi)
if *manifestFlag {
// Wrap the raw file entry in a proper manifest so both hashes get printed.
mroot = manifest{Entries: []manifest{mroot}}
}
entry, err = client.uploadFile(file, fi)
mroot = manifest{[]manifestEntry{entry}}
}
if err != nil {
log.Fatalln("upload failed:", err)
}
if *manifestFlag {
hash, err := client.uploadManifest(mroot)
if err != nil {
log.Fatalln("manifest upload failed:", err)
}
mroot.Hash = hash
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
hash, err := client.uploadManifest(mroot)
if err != nil {
log.Fatalln("manifest upload failed:", err)
}
fmt.Println(hash)
}
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
// Expands a file path
// 1. replace tilde with users home dir
// 2. expands embedded environment variables
// 3. cleans the path, e.g. /a/b/../c -> /a/c
// Note, it has limitations, e.g. ~someuser/tmp will not be expanded
func expandPath(p string) string {
if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") {
if home := homeDir(); home != "" {
p = home + p[1:]
}
}
return path.Clean(os.ExpandEnv(p))
}
func homeDir() string {
if home := os.Getenv("HOME"); home != "" {
return home
}
if usr, err := user.Current(); err == nil {
return usr.HomeDir
}
return ""
}
// client wraps interaction with the swarm HTTP gateway.
@ -88,24 +112,40 @@ type client struct {
}
// manifest is the JSON representation of a swarm manifest.
type manifest struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
Entries []manifest `json:"entries,omitempty"`
type manifestEntry struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
}
func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) {
// manifest is the JSON representation of a swarm manifest.
type manifest struct {
Entries []manifestEntry `json:"entries,omitempty"`
}
func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
hash, err := c.uploadFileContent(file, fi)
m := manifest{
m := manifestEntry{
Hash: hash,
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
}
return m, err
}
func (c *client) uploadDirectory(dir string) (manifest, error) {
func (c *client) uploadDirectory(dir string, defaultPath string) (manifest, error) {
dirm := manifest{}
if len(defaultPath) > 0 {
fi, err := os.Stat(defaultPath)
if err != nil {
log.Fatal(err)
}
entry, err := c.uploadFile(defaultPath, fi)
if err != nil {
log.Fatal(err)
}
entry.Path = ""
dirm.Entries = append(dirm.Entries, entry)
}
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() {

31
cmd/utils/cmd.go
View file

@ -18,12 +18,14 @@
package utils
import (
"compress/gzip"
"fmt"
"io"
"os"
"os/signal"
"regexp"
"runtime"
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
@ -133,7 +135,15 @@ func ImportChain(chain *core.BlockChain, fn string) error {
return err
}
defer fh.Close()
stream := rlp.NewStream(fh, 0)
var reader io.Reader = fh
if strings.HasSuffix(fn, ".gz") {
if reader, err = gzip.NewReader(reader); err != nil {
return err
}
}
stream := rlp.NewStream(reader, 0)
// Run actual the import.
blocks := make(types.Blocks, importBatchSize)
@ -195,10 +205,18 @@ func ExportChain(blockchain *core.BlockChain, fn string) error {
return err
}
defer fh.Close()
if err := blockchain.Export(fh); err != nil {
var writer io.Writer = fh
if strings.HasSuffix(fn, ".gz") {
writer = gzip.NewWriter(writer)
defer writer.(*gzip.Writer).Close()
}
if err := blockchain.Export(writer); err != nil {
return err
}
glog.Infoln("Exported blockchain to ", fn)
return nil
}
@ -210,7 +228,14 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
return err
}
defer fh.Close()
if err := blockchain.ExportN(fh, first, last); err != nil {
var writer io.Writer = fh
if strings.HasSuffix(fn, ".gz") {
writer = gzip.NewWriter(writer)
defer writer.(*gzip.Writer).Close()
}
if err := blockchain.ExportN(writer, first, last); err != nil {
return err
}
glog.Infoln("Exported blockchain to ", fn)

32
core/blockchain.go
View file

@ -601,7 +601,11 @@ func (self *BlockChain) procFutureBlocks() {
}
if len(blocks) > 0 {
types.BlockBy(types.Number).Sort(blocks)
self.InsertChain(blocks)
// Insert one by one as chain insertion needs contiguous ancestry between blocks
for i := range blocks {
self.InsertChain(blocks[i : i+1])
}
}
}
@ -675,6 +679,18 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty
// transaction and receipt data.
// XXX should this be moved to the test?
func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(blockChain); i++ {
if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() {
// Chain broke ancestry, log a messge (programming error) and skip insertion
failure := fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, blockChain[i-1].NumberU64(),
blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4])
glog.V(logger.Error).Info(failure.Error())
return 0, failure
}
}
// Pre-checks passed, start the block body and receipt imports
self.wg.Add(1)
defer self.wg.Done()
@ -843,6 +859,18 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err
// InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. It an error is returned
// it will return the index number of the failing block as well an error describing what went wrong (for possible errors see core/errors.go).
func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(chain); i++ {
if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() {
// Chain broke ancestry, log a messge (programming error) and skip insertion
failure := fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])",
i-1, chain[i-1].NumberU64(), chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
glog.V(logger.Error).Info(failure.Error())
return 0, failure
}
}
// Pre-checks passed, start the full block imports
self.wg.Add(1)
defer self.wg.Done()
@ -916,10 +944,8 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
}
self.reportBlock(block, nil, err)
return i, err
}
// Create a new statedb using the parent block and report an
// error if it fails.
switch {

11
core/headerchain.go
View file

@ -224,6 +224,17 @@ type WhCallback func(*types.Header) error
// of the header retrieval mechanisms already need to verfy nonces, as well as
// because nonces can be verified sparsely, not needing to check each.
func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, checkFreq int, writeHeader WhCallback) (int, error) {
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(chain); i++ {
if chain[i].Number.Uint64() != chain[i-1].Number.Uint64()+1 || chain[i].ParentHash != chain[i-1].Hash() {
// Chain broke ancestry, log a messge (programming error) and skip insertion
failure := fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])",
i-1, chain[i-1].Number.Uint64(), chain[i-1].Hash().Bytes()[:4], i, chain[i].Number.Uint64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash.Bytes()[:4])
glog.V(logger.Error).Info(failure.Error())
return 0, failure
}
}
// Collect some import statistics to report on
stats := struct{ processed, ignored int }{}
start := time.Now()

87
core/tx_pool.go
View file

@ -37,15 +37,14 @@ import (
var (
// Transaction Pool Errors
ErrInvalidSender = errors.New("Invalid sender")
ErrNonce = errors.New("Nonce too low")
ErrCheap = errors.New("Gas price too low for acceptance")
ErrBalance = errors.New("Insufficient balance")
ErrNonExistentAccount = errors.New("Account does not exist or account balance too low")
ErrInsufficientFunds = errors.New("Insufficient funds for gas * price + value")
ErrIntrinsicGas = errors.New("Intrinsic gas too low")
ErrGasLimit = errors.New("Exceeds block gas limit")
ErrNegativeValue = errors.New("Negative value")
ErrInvalidSender = errors.New("Invalid sender")
ErrNonce = errors.New("Nonce too low")
ErrCheap = errors.New("Gas price too low for acceptance")
ErrBalance = errors.New("Insufficient balance")
ErrInsufficientFunds = errors.New("Insufficient funds for gas * price + value")
ErrIntrinsicGas = errors.New("Intrinsic gas too low")
ErrGasLimit = errors.New("Exceeds block gas limit")
ErrNegativeValue = errors.New("Negative value")
)
var (
@ -124,6 +123,8 @@ func NewTxPool(config *params.ChainConfig, eventMux *event.TypeMux, currentState
quit: make(chan struct{}),
}
pool.resetState()
pool.wg.Add(2)
go pool.eventLoop()
go pool.expirationLoop()
@ -176,7 +177,7 @@ func (pool *TxPool) resetState() {
// any transactions that have been included in the block or
// have been invalidated because of another transaction (e.g.
// higher gas price)
pool.demoteUnexecutables()
pool.demoteUnexecutables(currentState)
// Update all accounts to the latest known pending nonce
for addr, list := range pool.pending {
@ -185,7 +186,7 @@ func (pool *TxPool) resetState() {
}
// Check the queue and move transactions over to the pending if possible
// or remove those that have become invalid
pool.promoteExecutables()
pool.promoteExecutables(currentState)
}
func (pool *TxPool) Stop() {
@ -237,21 +238,26 @@ func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common
// Pending retrieves all currently processable transactions, groupped by origin
// account and sorted by nonce. The returned transaction set is a copy and can be
// freely modified by calling code.
func (pool *TxPool) Pending() map[common.Address]types.Transactions {
func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) {
pool.mu.Lock()
defer pool.mu.Unlock()
state, err := pool.currentState()
if err != nil {
return nil, err
}
// check queue first
pool.promoteExecutables()
pool.promoteExecutables(state)
// invalidate any txs
pool.demoteUnexecutables()
pool.demoteUnexecutables(state)
pending := make(map[common.Address]types.Transactions)
for addr, list := range pool.pending {
pending[addr] = list.Flatten()
}
return pending
return pending, nil
}
// SetLocal marks a transaction as local, skipping gas price
@ -280,13 +286,6 @@ func (pool *TxPool) validateTx(tx *types.Transaction) error {
if err != nil {
return ErrInvalidSender
}
// Make sure the account exist. Non existent accounts
// haven't got funds and well therefor never pass.
if !currentState.Exist(from) {
return ErrNonExistentAccount
}
// Last but not least check for nonce errors
if currentState.GetNonce(from) > tx.Nonce() {
return ErrNonce
@ -372,10 +371,6 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) {
//
// Note, this method assumes the pool lock is held!
func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) {
// Init delayed since tx pool could have been started before any state sync
if pool.pendingState == nil {
pool.resetState()
}
// Try to insert the transaction into the pending queue
if pool.pending[addr] == nil {
pool.pending[addr] = newTxList(true)
@ -410,13 +405,19 @@ func (pool *TxPool) Add(tx *types.Transaction) error {
if err := pool.add(tx); err != nil {
return err
}
pool.promoteExecutables()
state, err := pool.currentState()
if err != nil {
return err
}
pool.promoteExecutables(state)
return nil
}
// AddBatch attempts to queue a batch of transactions.
func (pool *TxPool) AddBatch(txs []*types.Transaction) {
func (pool *TxPool) AddBatch(txs []*types.Transaction) error {
pool.mu.Lock()
defer pool.mu.Unlock()
@ -425,7 +426,15 @@ func (pool *TxPool) AddBatch(txs []*types.Transaction) {
glog.V(logger.Debug).Infoln("tx error:", err)
}
}
pool.promoteExecutables()
state, err := pool.currentState()
if err != nil {
return err
}
pool.promoteExecutables(state)
return nil
}
// Get returns a transaction if it is contained in the pool
@ -499,17 +508,7 @@ func (pool *TxPool) removeTx(hash common.Hash) {
// promoteExecutables moves transactions that have become processable from the
// future queue to the set of pending transactions. During this process, all
// invalidated transactions (low nonce, low balance) are deleted.
func (pool *TxPool) promoteExecutables() {
// Init delayed since tx pool could have been started before any state sync
if pool.pendingState == nil {
pool.resetState()
}
// Retrieve the current state to allow nonce and balance checking
state, err := pool.currentState()
if err != nil {
glog.Errorf("Could not get current state: %v", err)
return
}
func (pool *TxPool) promoteExecutables(state *state.StateDB) {
// Iterate over all accounts and promote any executable transactions
queued := uint64(0)
for addr, list := range pool.queue {
@ -645,13 +644,7 @@ func (pool *TxPool) promoteExecutables() {
// demoteUnexecutables removes invalid and processed transactions from the pools
// executable/pending queue and any subsequent transactions that become unexecutable
// are moved back into the future queue.
func (pool *TxPool) demoteUnexecutables() {
// Retrieve the current state to allow nonce and balance checking
state, err := pool.currentState()
if err != nil {
glog.V(logger.Info).Infoln("failed to get current state: %v", err)
return
}
func (pool *TxPool) demoteUnexecutables(state *state.StateDB) {
// Iterate over all accounts and demote any non-executable transactions
for addr, list := range pool.pending {
nonce := state.GetNonce(addr)

99
core/tx_pool_test.go
View file

@ -51,14 +51,84 @@ func deriveSender(tx *types.Transaction) (common.Address, error) {
return types.Sender(types.HomesteadSigner{}, tx)
}
// This test simulates a scenario where a new block is imported during a
// state reset and tests whether the pending state is in sync with the
// block head event that initiated the resetState().
func TestStateChangeDuringPoolReset(t *testing.T) {
var (
db, _ = ethdb.NewMemDatabase()
key, _ = crypto.GenerateKey()
address = crypto.PubkeyToAddress(key.PublicKey)
mux = new(event.TypeMux)
statedb, _ = state.New(common.Hash{}, db)
trigger = false
)
// setup pool with 2 transaction in it
statedb.SetBalance(address, new(big.Int).Mul(common.Big1, common.Ether))
tx0 := transaction(0, big.NewInt(100000), key)
tx1 := transaction(1, big.NewInt(100000), key)
// stateFunc is used multiple times to reset the pending state.
// when simulate is true it will create a state that indicates
// that tx0 and tx1 are included in the chain.
stateFunc := func() (*state.StateDB, error) {
// delay "state change" by one. The tx pool fetches the
// state multiple times and by delaying it a bit we simulate
// a state change between those fetches.
stdb := statedb
if trigger {
statedb, _ = state.New(common.Hash{}, db)
// simulate that the new head block included tx0 and tx1
statedb.SetNonce(address, 2)
statedb.SetBalance(address, new(big.Int).Mul(common.Big1, common.Ether))
trigger = false
}
return stdb, nil
}
gasLimitFunc := func() *big.Int { return big.NewInt(1000000000) }
txpool := NewTxPool(testChainConfig(), mux, stateFunc, gasLimitFunc)
txpool.resetState()
nonce := txpool.State().GetNonce(address)
if nonce != 0 {
t.Fatalf("Invalid nonce, want 0, got %d", nonce)
}
txpool.AddBatch(types.Transactions{tx0, tx1})
nonce = txpool.State().GetNonce(address)
if nonce != 2 {
t.Fatalf("Invalid nonce, want 2, got %d", nonce)
}
// trigger state change in the background
trigger = true
txpool.resetState()
pendingTx, err := txpool.Pending()
if err != nil {
t.Fatalf("Could not fetch pending transactions: %v", err)
}
for addr, txs := range pendingTx {
t.Logf("%0x: %d\n", addr, len(txs))
}
nonce = txpool.State().GetNonce(address)
if nonce != 2 {
t.Fatalf("Invalid nonce, want 2, got %d", nonce)
}
}
func TestInvalidTransactions(t *testing.T) {
pool, key := setupTxPool()
tx := transaction(0, big.NewInt(100), key)
if err := pool.Add(tx); err != ErrNonExistentAccount {
t.Error("expected", ErrNonExistentAccount)
}
from, _ := deriveSender(tx)
currentState, _ := pool.currentState()
currentState.AddBalance(from, big.NewInt(1))
@ -97,9 +167,10 @@ func TestTransactionQueue(t *testing.T) {
from, _ := deriveSender(tx)
currentState, _ := pool.currentState()
currentState.AddBalance(from, big.NewInt(1000))
pool.resetState()
pool.enqueueTx(tx.Hash(), tx)
pool.promoteExecutables()
pool.promoteExecutables(currentState)
if len(pool.pending) != 1 {
t.Error("expected valid txs to be 1 is", len(pool.pending))
}
@ -108,7 +179,7 @@ func TestTransactionQueue(t *testing.T) {
from, _ = deriveSender(tx)
currentState.SetNonce(from, 2)
pool.enqueueTx(tx.Hash(), tx)
pool.promoteExecutables()
pool.promoteExecutables(currentState)
if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok {
t.Error("expected transaction to be in tx pool")
}
@ -124,11 +195,13 @@ func TestTransactionQueue(t *testing.T) {
from, _ = deriveSender(tx1)
currentState, _ = pool.currentState()
currentState.AddBalance(from, big.NewInt(1000))
pool.resetState()
pool.enqueueTx(tx1.Hash(), tx1)
pool.enqueueTx(tx2.Hash(), tx2)
pool.enqueueTx(tx3.Hash(), tx3)
pool.promoteExecutables()
pool.promoteExecutables(currentState)
if len(pool.pending) != 1 {
t.Error("expected tx pool to be 1, got", len(pool.pending))
@ -225,7 +298,8 @@ func TestTransactionDoubleNonce(t *testing.T) {
if err := pool.add(tx2); err != nil {
t.Error("didn't expect error", err)
}
pool.promoteExecutables()
state, _ := pool.currentState()
pool.promoteExecutables(state)
if pool.pending[addr].Len() != 1 {
t.Error("expected 1 pending transactions, got", pool.pending[addr].Len())
}
@ -236,7 +310,7 @@ func TestTransactionDoubleNonce(t *testing.T) {
if err := pool.add(tx3); err != nil {
t.Error("didn't expect error", err)
}
pool.promoteExecutables()
pool.promoteExecutables(state)
if pool.pending[addr].Len() != 1 {
t.Error("expected 1 pending transactions, got", pool.pending[addr].Len())
}
@ -295,6 +369,7 @@ func TestRemovedTxEvent(t *testing.T) {
from, _ := deriveSender(tx)
currentState, _ := pool.currentState()
currentState.AddBalance(from, big.NewInt(1000000000000))
pool.resetState()
pool.eventMux.Post(RemovedTransactionEvent{types.Transactions{tx}})
pool.eventMux.Post(ChainHeadEvent{nil})
if pool.pending[from].Len() != 1 {
@ -452,6 +527,7 @@ func TestTransactionQueueAccountLimiting(t *testing.T) {
state, _ := pool.currentState()
state.AddBalance(account, big.NewInt(1000000))
pool.resetState()
// Keep queuing up transactions and make sure all above a limit are dropped
for i := uint64(1); i <= maxQueuedPerAccount+5; i++ {
@ -564,6 +640,7 @@ func TestTransactionPendingLimiting(t *testing.T) {
state, _ := pool.currentState()
state.AddBalance(account, big.NewInt(1000000))
pool.resetState()
// Keep queuing up transactions and make sure all above a limit are dropped
for i := uint64(0); i < maxQueuedPerAccount+5; i++ {
@ -733,7 +810,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) {
// Benchmark the speed of pool validation
b.ResetTimer()
for i := 0; i < b.N; i++ {
pool.demoteUnexecutables()
pool.demoteUnexecutables(state)
}
}
@ -757,7 +834,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) {
// Benchmark the speed of pool validation
b.ResetTimer()
for i := 0; i < b.N; i++ {
pool.promoteExecutables()
pool.promoteExecutables(state)
}
}

21
eth/api.go
View file

@ -18,6 +18,7 @@ package eth
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
@ -25,6 +26,7 @@ import (
"math/big"
"os"
"runtime"
"strings"
"time"
"github.com/ethereum/ethash"
@ -80,7 +82,7 @@ type PublicMinerAPI struct {
// NewPublicMinerAPI create a new PublicMinerAPI instance.
func NewPublicMinerAPI(e *Ethereum) *PublicMinerAPI {
agent := miner.NewRemoteAgent()
agent := miner.NewRemoteAgent(e.Pow())
e.Miner().Register(agent)
return &PublicMinerAPI{e, agent}
@ -217,8 +219,14 @@ func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) {
}
defer out.Close()
var writer io.Writer = out
if strings.HasSuffix(file, ".gz") {
writer = gzip.NewWriter(writer)
defer writer.(*gzip.Writer).Close()
}
// Export the blockchain
if err := api.eth.BlockChain().Export(out); err != nil {
if err := api.eth.BlockChain().Export(writer); err != nil {
return false, err
}
return true, nil
@ -243,8 +251,15 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) {
}
defer in.Close()
var reader io.Reader = in
if strings.HasSuffix(file, ".gz") {
if reader, err = gzip.NewReader(reader); err != nil {
return false, err
}
}
// Run actual the import in pre-configured batches
stream := rlp.NewStream(in, 0)
stream := rlp.NewStream(reader, 0)
blocks, index := make([]*types.Block, 0, 2500), 0
for batch := 0; ; batch++ {

11
eth/api_backend.go
View file

@ -131,15 +131,20 @@ func (b *EthApiBackend) RemoveTx(txHash common.Hash) {
b.eth.txPool.Remove(txHash)
}
func (b *EthApiBackend) GetPoolTransactions() types.Transactions {
func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) {
b.eth.txMu.Lock()
defer b.eth.txMu.Unlock()
pending, err := b.eth.txPool.Pending()
if err != nil {
return nil, err
}
var txs types.Transactions
for _, batch := range b.eth.txPool.Pending() {
for _, batch := range pending {
txs = append(txs, batch...)
}
return txs
return txs, nil
}
func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction {

2
eth/backend.go
View file

@ -104,6 +104,7 @@ type Config struct {
type LesServer interface {
Start(srvr *p2p.Server)
Synced()
Stop()
Protocols() []p2p.Protocol
}
@ -145,6 +146,7 @@ type Ethereum struct {
func (s *Ethereum) AddLesServer(ls LesServer) {
s.lesServer = ls
s.protocolManager.lesServer = ls
}
// New creates a new Ethereum object (including the

70
eth/filters/filter.go
View file

@ -54,6 +54,8 @@ type Filter struct {
// New creates a new filter which uses a bloom filter on blocks to figure out whether
// a particular block is interesting or not.
// MipMaps allow past blocks to be searched much more efficiently, but are not available
// to light clients.
func New(backend Backend, useMipMap bool) *Filter {
return &Filter{
backend: backend,
@ -85,8 +87,11 @@ func (f *Filter) SetTopics(topics [][]common.Hash) {
f.topics = topics
}
// Run filters logs with the current parameters set
func (f *Filter) Find(ctx context.Context) ([]*vm.Log, error) {
// FindOnce searches the blockchain for matching log entries, returning
// all matching entries from the first block that contains matches,
// updating the start point of the filter accordingly. If no results are
// found, a nil slice is returned.
func (f *Filter) FindOnce(ctx context.Context) ([]*vm.Log, error) {
head, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber)
if head == nil {
return nil, nil
@ -106,47 +111,69 @@ func (f *Filter) Find(ctx context.Context) ([]*vm.Log, error) {
// uses the mipmap bloom filters to check for fast inclusion and uses
// higher range probability in order to ensure at least a false positive
if !f.useMipMap || len(f.addresses) == 0 {
return f.getLogs(ctx, beginBlockNo, endBlockNo)
logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo)
f.begin = int64(blockNumber + 1)
return logs, err
}
return f.mipFind(beginBlockNo, endBlockNo, 0), nil
logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0)
f.begin = int64(blockNumber + 1)
return logs, nil
}
func (f *Filter) mipFind(start, end uint64, depth int) (logs []*vm.Log) {
// Run filters logs with the current parameters set
func (f *Filter) Find(ctx context.Context) (logs []*vm.Log, err error) {
for {
newLogs, err := f.FindOnce(ctx)
if len(newLogs) == 0 || err != nil {
return logs, err
}
logs = append(logs, newLogs...)
}
}
func (f *Filter) mipFind(start, end uint64, depth int) (logs []*vm.Log, blockNumber uint64) {
level := core.MIPMapLevels[depth]
// normalise numerator so we can work in level specific batches and
// work with the proper range checks
for num := start / level * level; num <= end; num += level {
// find addresses in bloom filters
bloom := core.GetMipmapBloom(f.db, num, level)
// Don't bother checking the first time through the loop - we're probably picking
// up where a previous run left off.
first := true
for _, addr := range f.addresses {
if bloom.TestBytes(addr[:]) {
if first || bloom.TestBytes(addr[:]) {
first = false
// range check normalised values and make sure that
// we're resolving the correct range instead of the
// normalised values.
start := uint64(math.Max(float64(num), float64(start)))
end := uint64(math.Min(float64(num+level-1), float64(end)))
if depth+1 == len(core.MIPMapLevels) {
l, _ := f.getLogs(context.Background(), start, end)
logs = append(logs, l...)
l, blockNumber, _ := f.getLogs(context.Background(), start, end)
if len(l) > 0 {
return l, blockNumber
}
} else {
logs = append(logs, f.mipFind(start, end, depth+1)...)
l, blockNumber := f.mipFind(start, end, depth+1)
if len(l) > 0 {
return l, blockNumber
}
}
// break so we don't check the same range for each
// possible address. Checks on multiple addresses
// are handled further down the stack.
break
}
}
}
return logs
return nil, end
}
func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*vm.Log, err error) {
func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*vm.Log, blockNumber uint64, err error) {
for i := start; i <= end; i++ {
header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(i))
blockNumber := rpc.BlockNumber(i)
header, err := f.backend.HeaderByNumber(ctx, blockNumber)
if header == nil || err != nil {
return logs, err
return logs, end, err
}
// Use bloom filtering to see if this block is interesting given the
@ -155,17 +182,20 @@ func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*vm.Log
// Get the logs of the block
receipts, err := f.backend.GetReceipts(ctx, header.Hash())
if err != nil {
return nil, err
return nil, end, err
}
var unfiltered []*vm.Log
for _, receipt := range receipts {
unfiltered = append(unfiltered, ([]*vm.Log)(receipt.Logs)...)
}
logs = append(logs, filterLogs(unfiltered, nil, nil, f.addresses, f.topics)...)
logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
if len(logs) > 0 {
return logs, uint64(blockNumber), nil
}
}
}
return logs, nil
return logs, end, nil
}
func includes(addresses []common.Address, a common.Address) bool {

4
eth/handler.go
View file

@ -87,6 +87,8 @@ type ProtocolManager struct {
quitSync chan struct{}
noMorePeers chan struct{}
lesServer LesServer
// wait group is used for graceful shutdowns during downloading
// and processing
wg sync.WaitGroup
@ -171,7 +173,7 @@ func NewProtocolManager(config *params.ChainConfig, fastSync bool, networkId int
return blockchain.CurrentBlock().NumberU64()
}
inserter := func(blocks types.Blocks) (int, error) {
atomic.StoreUint32(&manager.synced, 1) // Mark initial sync done on any fetcher import
manager.setSynced() // Mark initial sync done on any fetcher import
return manager.insertChain(blocks)
}
manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer)

8
eth/helper_test.go
View file

@ -93,7 +93,7 @@ type testTxPool struct {
// AddBatch appends a batch of transactions to the pool, and notifies any
// listeners if the addition channel is non nil
func (p *testTxPool) AddBatch(txs []*types.Transaction) {
func (p *testTxPool) AddBatch(txs []*types.Transaction) error {
p.lock.Lock()
defer p.lock.Unlock()
@ -101,10 +101,12 @@ func (p *testTxPool) AddBatch(txs []*types.Transaction) {
if p.added != nil {
p.added <- txs
}
return nil
}
// Pending returns all the transactions known to the pool
func (p *testTxPool) Pending() map[common.Address]types.Transactions {
func (p *testTxPool) Pending() (map[common.Address]types.Transactions, error) {
p.lock.RLock()
defer p.lock.RUnlock()
@ -116,7 +118,7 @@ func (p *testTxPool) Pending() map[common.Address]types.Transactions {
for _, batch := range batches {
sort.Sort(types.TxByNonce(batch))
}
return batches
return batches, nil
}
// newTestTransaction create a new dummy transaction.

4
eth/protocol.go
View file

@ -98,11 +98,11 @@ var errorToString = map[int]string{
type txPool interface {
// AddBatch should add the given transactions to the pool.
AddBatch([]*types.Transaction)
AddBatch([]*types.Transaction) error
// Pending should return pending transactions.
// The slice should be modifiable by the caller.
Pending() map[common.Address]types.Transactions
Pending() (map[common.Address]types.Transactions, error)
}
// statusData is the network packet for the status message.

12
eth/sync.go
View file

@ -46,7 +46,8 @@ type txsync struct {
// syncTransactions starts sending all currently pending transactions to the given peer.
func (pm *ProtocolManager) syncTransactions(p *peer) {
var txs types.Transactions
for _, batch := range pm.txpool.Pending() {
pending, _ := pm.txpool.Pending()
for _, batch := range pending {
txs = append(txs, batch...)
}
if len(txs) == 0 {
@ -180,7 +181,7 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil {
return
}
atomic.StoreUint32(&pm.synced, 1) // Mark initial sync done
pm.setSynced() // Mark initial sync done
// If fast sync was enabled, and we synced up, disable it
if atomic.LoadUint32(&pm.fastSync) == 1 {
@ -191,3 +192,10 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
}
}
}
// setSynced sets the synced flag and notifies the light server if present
func (pm *ProtocolManager) setSynced() {
if atomic.SwapUint32(&pm.synced, 1) == 0 && pm.lesServer != nil {
pm.lesServer.Synced()
}
}

235
ethstats/ethstats.go
View file

@ -25,6 +25,7 @@ import (
"regexp"
"runtime"
"strconv"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
@ -41,6 +42,10 @@ import (
"golang.org/x/net/websocket"
)
// historyUpdateRange is the number of blocks a node should report upon login or
// history request.
const historyUpdateRange = 50
// Service implements an Ethereum netstats reporting daemon that pushes local
// chain statistics up to a monitoring server.
type Service struct {
@ -53,6 +58,9 @@ type Service struct {
node string // Name of the node to display on the monitoring page
pass string // Password to authorize access to the monitoring page
host string // Remote address of the monitoring service
pongCh chan struct{} // Pong notifications are fed into this channel
histCh chan []uint64 // History request block numbers are fed into this channel
}
// New returns a monitoring service ready for stats reporting.
@ -65,11 +73,13 @@ func New(url string, ethServ *eth.Ethereum, lesServ *les.LightEthereum) (*Servic
}
// Assemble and return the stats service
return &Service{
eth: ethServ,
les: lesServ,
node: parts[1],
pass: parts[3],
host: parts[4],
eth: ethServ,
les: lesServ,
node: parts[1],
pass: parts[3],
host: parts[4],
pongCh: make(chan struct{}),
histCh: make(chan []uint64, 1),
}, nil
}
@ -115,7 +125,11 @@ func (s *Service) loop() {
// Loop reporting until termination
for {
// Establish a websocket connection to the server and authenticate the node
conn, err := websocket.Dial(fmt.Sprintf("wss://%s/api", s.host), "", "http://localhost/")
url := fmt.Sprintf("%s/api", s.host)
if !strings.Contains(url, "://") {
url = "wss://" + url
}
conn, err := websocket.Dial(url, "", "http://localhost/")
if err != nil {
glog.V(logger.Warn).Infof("Stats server unreachable: %v", err)
time.Sleep(10 * time.Second)
@ -130,22 +144,34 @@ func (s *Service) loop() {
time.Sleep(10 * time.Second)
continue
}
if err = s.report(in, out); err != nil {
go s.readLoop(conn, in)
// Send the initial stats so our node looks decent from the get go
if err = s.report(out); err != nil {
glog.V(logger.Warn).Infof("Initial stats report failed: %v", err)
conn.Close()
continue
}
if err = s.reportHistory(out, nil); err != nil {
glog.V(logger.Warn).Infof("History report failed: %v", err)
conn.Close()
continue
}
// Keep sending status updates until the connection breaks
fullReport := time.NewTicker(15 * time.Second)
for err == nil {
select {
case <-fullReport.C:
if err = s.report(in, out); err != nil {
if err = s.report(out); err != nil {
glog.V(logger.Warn).Infof("Full stats report failed: %v", err)
}
case head := <-headSub.Chan():
if head == nil { // node stopped
case list := <-s.histCh:
if err = s.reportHistory(out, list); err != nil {
glog.V(logger.Warn).Infof("Block history report failed: %v", err)
}
case head, ok := <-headSub.Chan():
if !ok { // node stopped
conn.Close()
return
}
@ -155,8 +181,8 @@ func (s *Service) loop() {
if err = s.reportPending(out); err != nil {
glog.V(logger.Warn).Infof("Post-block transaction stats report failed: %v", err)
}
case ev := <-txSub.Chan():
if ev == nil { // node stopped
case _, ok := <-txSub.Chan():
if !ok { // node stopped
conn.Close()
return
}
@ -178,6 +204,76 @@ func (s *Service) loop() {
}
}
// readLoop loops as long as the connection is alive and retrieves data packets
// from the network socket. If any of them match an active request, it forwards
// it, if they themselves are requests it initiates a reply, and lastly it drops
// unknown packets.
func (s *Service) readLoop(conn *websocket.Conn, in *json.Decoder) {
// If the read loop exists, close the connection
defer conn.Close()
for {
// Retrieve the next generic network packet and bail out on error
var msg map[string][]interface{}
if err := in.Decode(&msg); err != nil {
glog.V(logger.Warn).Infof("Failed to decode stats server message: %v", err)
return
}
if len(msg["emit"]) == 0 {
glog.V(logger.Warn).Infof("Stats server sent non-broadcast: %v", msg)
return
}
command, ok := msg["emit"][0].(string)
if !ok {
glog.V(logger.Warn).Infof("Invalid stats server message type: %v", msg["emit"][0])
return
}
// If the message is a ping reply, deliver (someone must be listening!)
if len(msg["emit"]) == 2 && command == "node-pong" {
select {
case s.pongCh <- struct{}{}:
// Pong delivered, continue listening
continue
default:
// Ping routine dead, abort
glog.V(logger.Warn).Infof("Stats server pinger seems to have died")
return
}
}
// If the message is a history request, forward to the event processor
if len(msg["emit"]) == 2 && command == "history" {
// Make sure the request is valid and doesn't crash us
request, ok := msg["emit"][1].(map[string]interface{})
if !ok {
glog.V(logger.Warn).Infof("Invalid history request: %v", msg["emit"][1])
return
}
list, ok := request["list"].([]interface{})
if !ok {
glog.V(logger.Warn).Infof("Invalid history block list: %v", request["list"])
return
}
// Convert the block number list to an integer list
numbers := make([]uint64, len(list))
for i, num := range list {
n, ok := num.(float64)
if !ok {
glog.V(logger.Warn).Infof("Invalid history block number: %v", num)
return
}
numbers[i] = uint64(n)
}
select {
case s.histCh <- numbers:
continue
default:
}
}
// Report anything else and continue
glog.V(logger.Info).Infof("Unknown stats message: %v", msg)
}
}
// nodeInfo is the collection of metainformation about a node that is displayed
// on the monitoring page.
type nodeInfo struct {
@ -190,6 +286,7 @@ type nodeInfo struct {
Os string `json:"os"`
OsVer string `json:"os_v"`
Client string `json:"client"`
History bool `json:"canUpdateHistory"`
}
// authMsg is the authentication infos needed to login to a monitoring server.
@ -224,6 +321,7 @@ func (s *Service) login(in *json.Decoder, out *json.Encoder) error {
Os: runtime.GOOS,
OsVer: runtime.GOARCH,
Client: "0.1.1",
History: true,
},
Secret: s.pass,
}
@ -244,8 +342,8 @@ func (s *Service) login(in *json.Decoder, out *json.Encoder) error {
// report collects all possible data to report and send it to the stats server.
// This should only be used on reconnects or rarely to avoid overloading the
// server. Use the individual methods for reporting subscribed events.
func (s *Service) report(in *json.Decoder, out *json.Encoder) error {
if err := s.reportLatency(in, out); err != nil {
func (s *Service) report(out *json.Encoder) error {
if err := s.reportLatency(out); err != nil {
return err
}
if err := s.reportBlock(out, nil); err != nil {
@ -262,7 +360,7 @@ func (s *Service) report(in *json.Decoder, out *json.Encoder) error {
// reportLatency sends a ping request to the server, measures the RTT time and
// finally sends a latency update.
func (s *Service) reportLatency(in *json.Decoder, out *json.Encoder) error {
func (s *Service) reportLatency(out *json.Encoder) error {
// Send the current time to the ethstats server
start := time.Now()
@ -276,9 +374,12 @@ func (s *Service) reportLatency(in *json.Decoder, out *json.Encoder) error {
return err
}
// Wait for the pong request to arrive back
var pong map[string][]interface{}
if err := in.Decode(&pong); err != nil || len(pong["emit"]) != 2 || pong["emit"][0].(string) != "node-pong" {
return errors.New("unexpected ping reply")
select {
case <-s.pongCh:
// Pong delivered, report the latency
case <-time.After(3 * time.Second):
// Ping timeout, abort
return errors.New("ping timed out")
}
// Send back the measured latency
latency := map[string][]interface{}{
@ -297,6 +398,7 @@ func (s *Service) reportLatency(in *json.Decoder, out *json.Encoder) error {
type blockStats struct {
Number *big.Int `json:"number"`
Hash common.Hash `json:"hash"`
Timestamp *big.Int `json:"timestamp"`
Miner common.Address `json:"miner"`
GasUsed *big.Int `json:"gasUsed"`
GasLimit *big.Int `json:"gasLimit"`
@ -330,9 +432,26 @@ func (s uncleStats) MarshalJSON() ([]byte, error) {
// reportBlock retrieves the current chain head and repors it to the stats server.
func (s *Service) reportBlock(out *json.Encoder, block *types.Block) error {
// Gather the head block infos from the local blockchain
// Assemble the block stats report and send it to the server
stats := map[string]interface{}{
"id": s.node,
"block": s.assembleBlockStats(block),
}
report := map[string][]interface{}{
"emit": []interface{}{"block", stats},
}
if err := out.Encode(report); err != nil {
return err
}
return nil
}
// assembleBlockStats retrieves any required metadata to report a single block
// and assembles the block stats. If block is nil, the current head is processed.
func (s *Service) assembleBlockStats(block *types.Block) *blockStats {
// Gather the block infos from the local blockchain
var (
head *types.Header
header *types.Header
td *big.Int
txs []*types.Transaction
uncles []*types.Header
@ -342,37 +461,77 @@ func (s *Service) reportBlock(out *json.Encoder, block *types.Block) error {
if block == nil {
block = s.eth.BlockChain().CurrentBlock()
}
head = block.Header()
td = s.eth.BlockChain().GetTd(head.Hash(), head.Number.Uint64())
header = block.Header()
td = s.eth.BlockChain().GetTd(header.Hash(), header.Number.Uint64())
txs = block.Transactions()
uncles = block.Uncles()
} else {
// Light nodes would need on-demand lookups for transactions/uncles, skip
if block != nil {
head = block.Header()
header = block.Header()
} else {
header = s.les.BlockChain().CurrentHeader()
}
td = s.les.BlockChain().GetTd(header.Hash(), header.Number.Uint64())
}
// Assemble and return the block stats
return &blockStats{
Number: header.Number,
Hash: header.Hash(),
Timestamp: header.Time,
Miner: header.Coinbase,
GasUsed: new(big.Int).Set(header.GasUsed),
GasLimit: new(big.Int).Set(header.GasLimit),
Diff: header.Difficulty.String(),
TotalDiff: td.String(),
Txs: txs,
Uncles: uncles,
}
}
// reportHistory retrieves the most recent batch of blocks and reports it to the
// stats server.
func (s *Service) reportHistory(out *json.Encoder, list []uint64) error {
// Figure out the indexes that need reporting
indexes := make([]uint64, 0, historyUpdateRange)
if len(list) > 0 {
// Specific indexes requested, send them back in particular
for _, idx := range list {
indexes = append(indexes, idx)
}
} else {
// No indexes requested, send back the top ones
var head *types.Header
if s.eth != nil {
head = s.eth.BlockChain().CurrentHeader()
} else {
head = s.les.BlockChain().CurrentHeader()
}
td = s.les.BlockChain().GetTd(head.Hash(), head.Number.Uint64())
start := head.Number.Int64() - historyUpdateRange
if start < 0 {
start = 0
}
for i := uint64(start); i <= head.Number.Uint64(); i++ {
indexes = append(indexes, i)
}
}
// Assemble the block stats report and send it to the server
// Gather the batch of blocks to report
history := make([]*blockStats, len(indexes))
for i, number := range indexes {
if s.eth != nil {
history[i] = s.assembleBlockStats(s.eth.BlockChain().GetBlockByNumber(number))
} else {
history[i] = s.assembleBlockStats(types.NewBlockWithHeader(s.les.BlockChain().GetHeaderByNumber(number)))
}
}
// Assemble the history report and send it to the server
stats := map[string]interface{}{
"id": s.node,
"block": &blockStats{
Number: head.Number,
Hash: head.Hash(),
Miner: head.Coinbase,
GasUsed: new(big.Int).Set(head.GasUsed),
GasLimit: new(big.Int).Set(head.GasLimit),
Diff: head.Difficulty.String(),
TotalDiff: td.String(),
Txs: txs,
Uncles: uncles,
},
"id": s.node,
"history": history,
}
report := map[string][]interface{}{
"emit": []interface{}{"block", stats},
"emit": []interface{}{"history", stats},
}
if err := out.Encode(report); err != nil {
return err

59
internal/ethapi/api.go
View file

@ -28,6 +28,7 @@ import (
"github.com/ethereum/ethash"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
@ -289,14 +290,14 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs
}
// signHash is a helper function that calculates a hash for the given message that can be
// safely used to calculate a signature from. The hash is calulcated with:
// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
func signHash(message string) []byte {
data := common.FromHex(message)
// Give context to the signed message. This prevents an adversery to sign a tx.
// It has no cryptographic purpose.
// safely used to calculate a signature from.
//
// The hash is calulcated as
// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
//
// This gives context to the signed message and prevents signing of transactions.
func signHash(data []byte) []byte {
msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
// Always hash, this prevents choosen plaintext attacks that can extract key information
return crypto.Keccak256([]byte(msg))
}
@ -306,13 +307,8 @@ func signHash(message string) []byte {
// The key used to calculate the signature is decrypted with the given password.
//
// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_sign
func (s *PrivateAccountAPI) Sign(ctx context.Context, message string, addr common.Address, passwd string) (string, error) {
hash := signHash(message)
signature, err := s.b.AccountManager().SignWithPassphrase(addr, passwd, hash)
if err != nil {
return "0x", err
}
return common.ToHex(signature), nil
func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) {
return s.b.AccountManager().SignWithPassphrase(addr, passwd, signHash(data))
}
// EcRecover returns the address for the account that was used to create the signature.
@ -322,29 +318,20 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, message string, addr commo
// addr = ecrecover(hash, signature)
//
// https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover
func (s *PrivateAccountAPI) EcRecover(ctx context.Context, message string, signature string) (common.Address, error) {
var (
hash = signHash(message)
sig = common.FromHex(signature)
)
func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) {
if len(sig) != 65 {
return common.Address{}, fmt.Errorf("signature must be 65 bytes long")
}
// see crypto.Ecrecover description
if sig[64] == 27 || sig[64] == 28 {
sig[64] -= 27
}
rpk, err := crypto.Ecrecover(hash, sig)
rpk, err := crypto.Ecrecover(signHash(data), sig)
if err != nil {
return common.Address{}, err
}
pubKey := crypto.ToECDSAPub(rpk)
recoveredAddr := crypto.PubkeyToAddress(*pubKey)
return recoveredAddr, nil
}
@ -1116,10 +1103,8 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod
// The account associated with addr must be unlocked.
//
// https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_sign
func (s *PublicTransactionPoolAPI) Sign(addr common.Address, message string) (string, error) {
hash := signHash(message)
signature, err := s.b.AccountManager().SignEthereum(addr, hash)
return common.ToHex(signature), err
func (s *PublicTransactionPoolAPI) Sign(addr common.Address, data hexutil.Bytes) (hexutil.Bytes, error) {
return s.b.AccountManager().SignEthereum(addr, signHash(data))
}
// SignTransactionArgs represents the arguments to sign a transaction.
@ -1273,8 +1258,12 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Sig
// PendingTransactions returns the transactions that are in the transaction pool and have a from address that is one of
// the accounts this node manages.
func (s *PublicTransactionPoolAPI) PendingTransactions() []*RPCTransaction {
pending := s.b.GetPoolTransactions()
func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, error) {
pending, err := s.b.GetPoolTransactions()
if err != nil {
return nil, err
}
transactions := make([]*RPCTransaction, 0, len(pending))
for _, tx := range pending {
var signer types.Signer = types.HomesteadSigner{}
@ -1286,13 +1275,17 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() []*RPCTransaction {
transactions = append(transactions, newRPCPendingTransaction(tx))
}
}
return transactions
return transactions, nil
}
// Resend accepts an existing transaction and a new gas price and limit. It will remove the given transaction from the
// pool and reinsert it with the new gas price and limit.
func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, tx Tx, gasPrice, gasLimit *rpc.HexNumber) (common.Hash, error) {
pending := s.b.GetPoolTransactions()
pending, err := s.b.GetPoolTransactions()
if err != nil {
return common.Hash{}, err
}
for _, p := range pending {
var signer types.Signer = types.HomesteadSigner{}
if p.Protected() {

2
internal/ethapi/backend.go
View file

@ -55,7 +55,7 @@ type Backend interface {
// TxPool API
SendTx(ctx context.Context, signedTx *types.Transaction) error
RemoveTx(txHash common.Hash)
GetPoolTransactions() types.Transactions
GetPoolTransactions() (types.Transactions, error)
GetPoolTransaction(txHash common.Hash) *types.Transaction
GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
Stats() (pending int, queued int)

2
les/api_backend.go
View file

@ -110,7 +110,7 @@ func (b *LesApiBackend) RemoveTx(txHash common.Hash) {
b.eth.txPool.RemoveTx(txHash)
}
func (b *LesApiBackend) GetPoolTransactions() types.Transactions {
func (b *LesApiBackend) GetPoolTransactions() (types.Transactions, error) {
return b.eth.txPool.GetTransactions()
}

776
les/fetcher.go
View file

@ -23,135 +23,374 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/light"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
)
const (
blockDelayTimeout = time.Second * 10 // timeout for a peer to announce a head that has already been confirmed by others
maxNodeCount = 20 // maximum number of fetcherTreeNode entries remembered for each peer
)
// lightFetcher
type lightFetcher struct {
pm *ProtocolManager
odr *LesOdr
chain BlockChain
chain *light.LightChain
headAnnouncedMu sync.Mutex
headAnnouncedBy map[common.Hash][]*peer
currentTd *big.Int
deliverChn chan fetchResponse
reqMu sync.RWMutex
requested map[uint64]fetchRequest
timeoutChn chan uint64
notifyChn chan bool // true if initiated from outside
syncing bool
syncDone chan struct{}
maxConfirmedTd *big.Int
peers map[*peer]*fetcherPeerInfo
lastUpdateStats *updateStatsEntry
lock sync.Mutex // qwerqwerqwe
deliverChn chan fetchResponse
reqMu sync.RWMutex
requested map[uint64]fetchRequest
timeoutChn chan uint64
requestChn chan bool // true if initiated from outside
syncing bool
syncDone chan *peer
}
// fetcherPeerInfo holds fetcher-specific information about each active peer
type fetcherPeerInfo struct {
root, lastAnnounced *fetcherTreeNode
nodeCnt int
confirmedTd *big.Int
bestConfirmed *fetcherTreeNode
nodeByHash map[common.Hash]*fetcherTreeNode
firstUpdateStats *updateStatsEntry
}
// fetcherTreeNode is a node of a tree that holds information about blocks recently
// announced and confirmed by a certain peer. Each new announce message from a peer
// adds nodes to the tree, based on the previous announced head and the reorg depth.
// There are three possible states for a tree node:
// - announced: not downloaded (known) yet, but we know its head, number and td
// - intermediate: not known, hash and td are empty, they are filled out when it becomes known
// - known: both announced by this peer and downloaded (from any peer).
// This structure makes it possible to always know which peer has a certain block,
// which is necessary for selecting a suitable peer for ODR requests and also for
// canonizing new heads. It also helps to always download the minimum necessary
// amount of headers with a single request.
type fetcherTreeNode struct {
hash common.Hash
number uint64
td *big.Int
known, requested bool
parent *fetcherTreeNode
children []*fetcherTreeNode
}
// fetchRequest represents a header download request
type fetchRequest struct {
hash common.Hash
amount uint64
peer *peer
hash common.Hash
amount uint64
peer *peer
sent mclock.AbsTime
timeout bool
}
// fetchResponse represents a header download response
type fetchResponse struct {
reqID uint64
headers []*types.Header
peer *peer
}
// newLightFetcher creates a new light fetcher
func newLightFetcher(pm *ProtocolManager) *lightFetcher {
f := &lightFetcher{
pm: pm,
chain: pm.blockchain,
odr: pm.odr,
headAnnouncedBy: make(map[common.Hash][]*peer),
deliverChn: make(chan fetchResponse, 100),
requested: make(map[uint64]fetchRequest),
timeoutChn: make(chan uint64),
notifyChn: make(chan bool, 100),
syncDone: make(chan struct{}),
currentTd: big.NewInt(0),
pm: pm,
chain: pm.blockchain.(*light.LightChain),
odr: pm.odr,
peers: make(map[*peer]*fetcherPeerInfo),
deliverChn: make(chan fetchResponse, 100),
requested: make(map[uint64]fetchRequest),
timeoutChn: make(chan uint64),
requestChn: make(chan bool, 100),
syncDone: make(chan *peer),
maxConfirmedTd: big.NewInt(0),
}
go f.syncLoop()
return f
}
func (f *lightFetcher) notify(p *peer, head *announceData) {
var headHash common.Hash
if head == nil {
// initial notify
headHash = p.Head()
} else {
if core.GetTd(f.pm.chainDb, head.Hash, head.Number) != nil {
head.haveHeaders = head.Number
// syncLoop is the main event loop of the light fetcher
func (f *lightFetcher) syncLoop() {
f.pm.wg.Add(1)
defer f.pm.wg.Done()
requestStarted := false
for {
select {
case <-f.pm.quitSync:
return
// when a new announce is received, request loop keeps running until
// no further requests are necessary or possible
case newAnnounce := <-f.requestChn:
f.lock.Lock()
s := requestStarted
requestStarted = false
if !f.syncing && !(newAnnounce && s) {
if peer, node, amount := f.nextRequest(); node != nil {
requestStarted = true
reqID, started := f.request(peer, node, amount)
if started {
go func() {
time.Sleep(softRequestTimeout)
f.reqMu.Lock()
req, ok := f.requested[reqID]
if ok {
req.timeout = true
f.requested[reqID] = req
}
f.reqMu.Unlock()
// keep starting new requests while possible
f.requestChn <- false
}()
}
}
}
f.lock.Unlock()
case reqID := <-f.timeoutChn:
f.reqMu.Lock()
req, ok := f.requested[reqID]
if ok {
delete(f.requested, reqID)
}
f.reqMu.Unlock()
if ok {
f.pm.serverPool.adjustResponseTime(req.peer.poolEntry, time.Duration(mclock.Now()-req.sent), true)
glog.V(logger.Debug).Infof("hard timeout by peer %v", req.peer.id)
go f.pm.removePeer(req.peer.id)
}
case resp := <-f.deliverChn:
f.reqMu.Lock()
req, ok := f.requested[resp.reqID]
if ok && req.peer != resp.peer {
ok = false
}
if ok {
delete(f.requested, resp.reqID)
}
f.reqMu.Unlock()
if ok {
f.pm.serverPool.adjustResponseTime(req.peer.poolEntry, time.Duration(mclock.Now()-req.sent), req.timeout)
}
f.lock.Lock()
if !ok || !(f.syncing || f.processResponse(req, resp)) {
glog.V(logger.Debug).Infof("failed processing response by peer %v", resp.peer.id)
go f.pm.removePeer(resp.peer.id)
}
f.lock.Unlock()
case p := <-f.syncDone:
f.lock.Lock()
glog.V(logger.Debug).Infof("done synchronising with peer %v", p.id)
f.checkSyncedHeaders(p)
f.syncing = false
f.lock.Unlock()
}
//fmt.Println("notify", p.id, head.Number, head.ReorgDepth, head.haveHeaders)
if !p.addNotify(head) {
//fmt.Println("addNotify fail")
f.pm.removePeer(p.id)
}
headHash = head.Hash
}
f.headAnnouncedMu.Lock()
f.headAnnouncedBy[headHash] = append(f.headAnnouncedBy[headHash], p)
f.headAnnouncedMu.Unlock()
f.notifyChn <- true
}
func (f *lightFetcher) gotHeader(header *types.Header) {
f.headAnnouncedMu.Lock()
defer f.headAnnouncedMu.Unlock()
// addPeer adds a new peer to the fetcher's peer set
func (f *lightFetcher) addPeer(p *peer) {
p.lock.Lock()
p.hasBlock = func(hash common.Hash, number uint64) bool {
return f.peerHasBlock(p, hash, number)
}
p.lock.Unlock()
hash := header.Hash()
peerList := f.headAnnouncedBy[hash]
if peerList == nil {
f.lock.Lock()
defer f.lock.Unlock()
f.peers[p] = &fetcherPeerInfo{nodeByHash: make(map[common.Hash]*fetcherTreeNode)}
}
// removePeer removes a new peer from the fetcher's peer set
func (f *lightFetcher) removePeer(p *peer) {
p.lock.Lock()
p.hasBlock = nil
p.lock.Unlock()
f.lock.Lock()
defer f.lock.Unlock()
// check for potential timed out block delay statistics
f.checkUpdateStats(p, nil)
delete(f.peers, p)
}
// announce processes a new announcement message received from a peer, adding new
// nodes to the peer's block tree and removing old nodes if necessary
func (f *lightFetcher) announce(p *peer, head *announceData) {
f.lock.Lock()
defer f.lock.Unlock()
glog.V(logger.Debug).Infof("received announce from peer %v #%d %016x reorg: %d", p.id, head.Number, head.Hash[:8], head.ReorgDepth)
fp := f.peers[p]
if fp == nil {
glog.V(logger.Debug).Infof("announce: unknown peer")
return
}
number := header.Number.Uint64()
td := core.GetTd(f.pm.chainDb, hash, number)
for _, peer := range peerList {
peer.lock.Lock()
ok := peer.gotHeader(hash, number, td)
peer.lock.Unlock()
if !ok {
//fmt.Println("gotHeader fail")
f.pm.removePeer(peer.id)
}
}
delete(f.headAnnouncedBy, hash)
}
func (f *lightFetcher) nextRequest() (*peer, *announceData) {
var bestPeer *peer
bestTd := f.currentTd
for _, peer := range f.pm.peers.AllPeers() {
peer.lock.RLock()
if !peer.headInfo.requested && (peer.headInfo.Td.Cmp(bestTd) > 0 ||
(bestPeer != nil && peer.headInfo.Td.Cmp(bestTd) == 0 && peer.headInfo.haveHeaders > bestPeer.headInfo.haveHeaders)) {
bestPeer = peer
bestTd = peer.headInfo.Td
if fp.lastAnnounced != nil && head.Td.Cmp(fp.lastAnnounced.td) <= 0 {
// announced tds should be strictly monotonic
glog.V(logger.Debug).Infof("non-monotonic Td from peer %v", p.id)
go f.pm.removePeer(p.id)
return
}
n := fp.lastAnnounced
for i := uint64(0); i < head.ReorgDepth; i++ {
if n == nil {
break
}
peer.lock.RUnlock()
n = n.parent
}
if bestPeer == nil {
return nil, nil
}
bestPeer.lock.Lock()
res := bestPeer.headInfo
res.requested = true
bestPeer.lock.Unlock()
for _, peer := range f.pm.peers.AllPeers() {
if peer != bestPeer {
peer.lock.Lock()
if peer.headInfo.Hash == bestPeer.headInfo.Hash && peer.headInfo.haveHeaders == bestPeer.headInfo.haveHeaders {
peer.headInfo.requested = true
if n != nil {
// n is now the reorg common ancestor, add a new branch of nodes
// check if the node count is too high to add new nodes
locked := false
for uint64(fp.nodeCnt)+head.Number-n.number > maxNodeCount && fp.root != nil {
if !locked {
f.chain.LockChain()
defer f.chain.UnlockChain()
locked = true
}
peer.lock.Unlock()
// if one of root's children is canonical, keep it, delete other branches and root itself
var newRoot *fetcherTreeNode
for i, nn := range fp.root.children {
if core.GetCanonicalHash(f.pm.chainDb, nn.number) == nn.hash {
fp.root.children = append(fp.root.children[:i], fp.root.children[i+1:]...)
nn.parent = nil
newRoot = nn
break
}
}
fp.deleteNode(fp.root)
if n == fp.root {
n = newRoot
}
fp.root = newRoot
if newRoot == nil || !f.checkKnownNode(p, newRoot) {
fp.bestConfirmed = nil
fp.confirmedTd = nil
}
if n == nil {
break
}
}
if n != nil {
for n.number < head.Number {
nn := &fetcherTreeNode{number: n.number + 1, parent: n}
n.children = append(n.children, nn)
n = nn
fp.nodeCnt++
}
n.hash = head.Hash
n.td = head.Td
fp.nodeByHash[n.hash] = n
}
}
return bestPeer, res
if n == nil {
// could not find reorg common ancestor or had to delete entire tree, a new root and a resync is needed
if fp.root != nil {
fp.deleteNode(fp.root)
}
n = &fetcherTreeNode{hash: head.Hash, number: head.Number, td: head.Td}
fp.root = n
fp.nodeCnt++
fp.nodeByHash[n.hash] = n
fp.bestConfirmed = nil
fp.confirmedTd = nil
}
f.checkKnownNode(p, n)
p.lock.Lock()
p.headInfo = head
fp.lastAnnounced = n
p.lock.Unlock()
f.checkUpdateStats(p, nil)
f.requestChn <- true
}
func (f *lightFetcher) deliverHeaders(reqID uint64, headers []*types.Header) {
f.deliverChn <- fetchResponse{reqID: reqID, headers: headers}
// peerHasBlock returns true if we can assume the peer knows the given block
// based on its announcements
func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bool {
f.lock.Lock()
defer f.lock.Unlock()
fp := f.peers[p]
if fp == nil || fp.root == nil {
return false
}
if number >= fp.root.number {
// it is recent enough that if it is known, is should be in the peer's block tree
return fp.nodeByHash[hash] != nil
}
f.chain.LockChain()
defer f.chain.UnlockChain()
// if it's older than the peer's block tree root but it's in the same canonical chain
// than the root, we can still be sure the peer knows it
return core.GetCanonicalHash(f.pm.chainDb, fp.root.number) == fp.root.hash && core.GetCanonicalHash(f.pm.chainDb, number) == hash
}
// request initiates a header download request from a certain peer
func (f *lightFetcher) request(p *peer, n *fetcherTreeNode, amount uint64) (uint64, bool) {
fp := f.peers[p]
if fp == nil {
glog.V(logger.Debug).Infof("request: unknown peer")
return 0, false
}
if fp.bestConfirmed == nil || fp.root == nil || !f.checkKnownNode(p, fp.root) {
f.syncing = true
go func() {
glog.V(logger.Debug).Infof("synchronising with peer %v", p.id)
f.pm.synchronise(p)
f.syncDone <- p
}()
return 0, false
}
reqID := getNextReqID()
n.requested = true
cost := p.GetRequestCost(GetBlockHeadersMsg, int(amount))
p.fcServer.SendRequest(reqID, cost)
f.reqMu.Lock()
f.requested[reqID] = fetchRequest{hash: n.hash, amount: amount, peer: p, sent: mclock.Now()}
f.reqMu.Unlock()
go p.RequestHeadersByHash(reqID, cost, n.hash, int(amount), 0, true)
go func() {
time.Sleep(hardRequestTimeout)
f.timeoutChn <- reqID
}()
return reqID, true
}
// requestAmount calculates the amount of headers to be downloaded starting
// from a certain head backwards
func (f *lightFetcher) requestAmount(p *peer, n *fetcherTreeNode) uint64 {
amount := uint64(0)
nn := n
for nn != nil && !f.checkKnownNode(p, nn) {
nn = nn.parent
amount++
}
if nn == nil {
amount = n.number
}
return amount
}
// requestedID tells if a certain reqID has been requested by the fetcher
func (f *lightFetcher) requestedID(reqID uint64) bool {
f.reqMu.RLock()
_, ok := f.requested[reqID]
@ -159,36 +398,51 @@ func (f *lightFetcher) requestedID(reqID uint64) bool {
return ok
}
func (f *lightFetcher) request(p *peer, block *announceData) {
//fmt.Println("request", p.id, block.Number, block.haveHeaders)
amount := block.Number - block.haveHeaders
if amount == 0 {
return
// nextRequest selects the peer and announced head to be requested next, amount
// to be downloaded starting from the head backwards is also returned
func (f *lightFetcher) nextRequest() (*peer, *fetcherTreeNode, uint64) {
var (
bestHash common.Hash
bestAmount uint64
)
bestTd := f.maxConfirmedTd
for p, fp := range f.peers {
for hash, n := range fp.nodeByHash {
if !f.checkKnownNode(p, n) && !n.requested && (bestTd == nil || n.td.Cmp(bestTd) >= 0) {
amount := f.requestAmount(p, n)
if bestTd == nil || n.td.Cmp(bestTd) > 0 || amount < bestAmount {
bestHash = hash
bestAmount = amount
bestTd = n.td
}
}
}
}
if amount > 100 {
f.syncing = true
go func() {
//fmt.Println("f.pm.synchronise(p)")
f.pm.synchronise(p)
//fmt.Println("sync done")
f.syncDone <- struct{}{}
}()
return
if bestTd == f.maxConfirmedTd {
return nil, nil, 0
}
reqID := f.odr.getNextReqID()
f.reqMu.Lock()
f.requested[reqID] = fetchRequest{hash: block.Hash, amount: amount, peer: p}
f.reqMu.Unlock()
cost := p.GetRequestCost(GetBlockHeadersMsg, int(amount))
p.fcServer.SendRequest(reqID, cost)
go p.RequestHeadersByHash(reqID, cost, block.Hash, int(amount), 0, true)
go func() {
time.Sleep(hardRequestTimeout)
f.timeoutChn <- reqID
}()
peer := f.pm.serverPool.selectPeer(func(p *peer) (bool, uint64) {
fp := f.peers[p]
if fp == nil || fp.nodeByHash[bestHash] == nil {
return false, 0
}
return true, p.fcServer.CanSend(p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)))
})
var node *fetcherTreeNode
if peer != nil {
node = f.peers[peer].nodeByHash[bestHash]
}
return peer, node, bestAmount
}
// deliverHeaders delivers header download request responses for processing
func (f *lightFetcher) deliverHeaders(peer *peer, reqID uint64, headers []*types.Header) {
f.deliverChn <- fetchResponse{reqID: reqID, headers: headers, peer: peer}
}
// processResponse processes header download request responses
func (f *lightFetcher) processResponse(req fetchRequest, resp fetchResponse) bool {
if uint64(len(resp.headers)) != req.amount || resp.headers[0].Hash() != req.hash {
return false
@ -200,96 +454,248 @@ func (f *lightFetcher) processResponse(req fetchRequest, resp fetchResponse) boo
if _, err := f.chain.InsertHeaderChain(headers, 1); err != nil {
return false
}
for _, header := range headers {
td := core.GetTd(f.pm.chainDb, header.Hash(), header.Number.Uint64())
tds := make([]*big.Int, len(headers))
for i, header := range headers {
td := f.chain.GetTd(header.Hash(), header.Number.Uint64())
if td == nil {
return false
}
if td.Cmp(f.currentTd) > 0 {
f.currentTd = td
}
f.gotHeader(header)
tds[i] = td
}
f.newHeaders(headers, tds)
return true
}
func (f *lightFetcher) checkSyncedHeaders() {
//fmt.Println("checkSyncedHeaders()")
for _, peer := range f.pm.peers.AllPeers() {
peer.lock.Lock()
h := peer.firstHeadInfo
remove := false
loop:
for h != nil {
if td := core.GetTd(f.pm.chainDb, h.Hash, h.Number); td != nil {
//fmt.Println(" found", h.Number)
ok := peer.gotHeader(h.Hash, h.Number, td)
if !ok {
remove = true
break loop
}
if td.Cmp(f.currentTd) > 0 {
f.currentTd = td
// newHeaders updates the block trees of all active peers according to a newly
// downloaded and validated batch or headers
func (f *lightFetcher) newHeaders(headers []*types.Header, tds []*big.Int) {
var maxTd *big.Int
for p, fp := range f.peers {
if !f.checkAnnouncedHeaders(fp, headers, tds) {
glog.V(logger.Debug).Infof("announce inconsistency by peer %v", p.id)
go f.pm.removePeer(p.id)
}
if fp.confirmedTd != nil && (maxTd == nil || maxTd.Cmp(fp.confirmedTd) > 0) {
maxTd = fp.confirmedTd
}
}
if maxTd != nil {
f.updateMaxConfirmedTd(maxTd)
}
}
// checkAnnouncedHeaders updates peer's block tree if necessary after validating
// a batch of headers. It searches for the latest header in the batch that has a
// matching tree node (if any), and if it has not been marked as known already,
// sets it and its parents to known (even those which are older than the currently
// validated ones). Return value shows if all hashes, numbers and Tds matched
// correctly to the announced values (otherwise the peer should be dropped).
func (f *lightFetcher) checkAnnouncedHeaders(fp *fetcherPeerInfo, headers []*types.Header, tds []*big.Int) bool {
var (
n *fetcherTreeNode
header *types.Header
td *big.Int
)
for i := len(headers) - 1; ; i-- {
if i < 0 {
if n == nil {
// no more headers and nothing to match
return true
}
// we ran out of recently delivered headers but have not reached a node known by this peer yet, continue matching
td = f.chain.GetTd(header.ParentHash, header.Number.Uint64()-1)
header = f.chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
} else {
header = headers[i]
td = tds[i]
}
hash := header.Hash()
number := header.Number.Uint64()
if n == nil {
n = fp.nodeByHash[hash]
}
if n != nil {
if n.td == nil {
// node was unannounced
if nn := fp.nodeByHash[hash]; nn != nil {
// if there was already a node with the same hash, continue there and drop this one
nn.children = append(nn.children, n.children...)
n.children = nil
fp.deleteNode(n)
n = nn
} else {
n.hash = hash
n.td = td
fp.nodeByHash[hash] = n
}
}
h = h.next
}
peer.lock.Unlock()
if remove {
//fmt.Println("checkSync fail")
f.pm.removePeer(peer.id)
// check if it matches the header
if n.hash != hash || n.number != number || n.td.Cmp(td) != 0 {
// peer has previously made an invalid announcement
return false
}
if n.known {
// we reached a known node that matched our expectations, return with success
return true
}
n.known = true
if fp.confirmedTd == nil || td.Cmp(fp.confirmedTd) > 0 {
fp.confirmedTd = td
fp.bestConfirmed = n
}
n = n.parent
if n == nil {
return true
}
}
}
}
func (f *lightFetcher) syncLoop() {
f.pm.wg.Add(1)
defer f.pm.wg.Done()
// checkSyncedHeaders updates peer's block tree after synchronisation by marking
// downloaded headers as known. If none of the announced headers are found after
// syncing, the peer is dropped.
func (f *lightFetcher) checkSyncedHeaders(p *peer) {
fp := f.peers[p]
if fp == nil {
glog.V(logger.Debug).Infof("checkSyncedHeaders: unknown peer")
return
}
n := fp.lastAnnounced
var td *big.Int
for n != nil {
if td = f.chain.GetTd(n.hash, n.number); td != nil {
break
}
n = n.parent
}
// now n is the latest downloaded header after syncing
if n == nil {
glog.V(logger.Debug).Infof("synchronisation failed with peer %v", p.id)
go f.pm.removePeer(p.id)
} else {
header := f.chain.GetHeader(n.hash, n.number)
f.newHeaders([]*types.Header{header}, []*big.Int{td})
}
}
srtoNotify := false
// checkKnownNode checks if a block tree node is known (downloaded and validated)
// If it was not known previously but found in the database, sets its known flag
func (f *lightFetcher) checkKnownNode(p *peer, n *fetcherTreeNode) bool {
if n.known {
return true
}
td := f.chain.GetTd(n.hash, n.number)
if td == nil {
return false
}
fp := f.peers[p]
if fp == nil {
glog.V(logger.Debug).Infof("checkKnownNode: unknown peer")
return false
}
header := f.chain.GetHeader(n.hash, n.number)
if !f.checkAnnouncedHeaders(fp, []*types.Header{header}, []*big.Int{td}) {
glog.V(logger.Debug).Infof("announce inconsistency by peer %v", p.id)
go f.pm.removePeer(p.id)
}
if fp.confirmedTd != nil {
f.updateMaxConfirmedTd(fp.confirmedTd)
}
return n.known
}
// deleteNode deletes a node and its child subtrees from a peer's block tree
func (fp *fetcherPeerInfo) deleteNode(n *fetcherTreeNode) {
if n.parent != nil {
for i, nn := range n.parent.children {
if nn == n {
n.parent.children = append(n.parent.children[:i], n.parent.children[i+1:]...)
break
}
}
}
for {
select {
case <-f.pm.quitSync:
if n.td != nil {
delete(fp.nodeByHash, n.hash)
}
fp.nodeCnt--
if len(n.children) == 0 {
return
case ext := <-f.notifyChn:
//fmt.Println("<-f.notifyChn", f.syncing, ext, srtoNotify)
s := srtoNotify
srtoNotify = false
if !f.syncing && !(ext && s) {
if p, r := f.nextRequest(); r != nil {
srtoNotify = true
go func() {
time.Sleep(softRequestTimeout)
f.notifyChn <- false
}()
f.request(p, r)
}
}
for i, nn := range n.children {
if i == 0 {
n = nn
} else {
fp.deleteNode(nn)
}
case reqID := <-f.timeoutChn:
f.reqMu.Lock()
req, ok := f.requested[reqID]
if ok {
delete(f.requested, reqID)
}
f.reqMu.Unlock()
if ok {
//fmt.Println("hard timeout")
f.pm.removePeer(req.peer.id)
}
case resp := <-f.deliverChn:
//fmt.Println("<-f.deliverChn", f.syncing)
f.reqMu.Lock()
req, ok := f.requested[resp.reqID]
delete(f.requested, resp.reqID)
f.reqMu.Unlock()
if !ok || !(f.syncing || f.processResponse(req, resp)) {
//fmt.Println("processResponse fail")
f.pm.removePeer(req.peer.id)
}
case <-f.syncDone:
//fmt.Println("<-f.syncDone", f.syncing)
f.checkSyncedHeaders()
f.syncing = false
}
}
}
// updateStatsEntry items form a linked list that is expanded with a new item every time a new head with a higher Td
// than the previous one has been downloaded and validated. The list contains a series of maximum confirmed Td values
// and the time these values have been confirmed, both increasing monotonically. A maximum confirmed Td is calculated
// both globally for all peers and also for each individual peer (meaning that the given peer has announced the head
// and it has also been downloaded from any peer, either before or after the given announcement).
// The linked list has a global tail where new confirmed Td entries are added and a separate head for each peer,
// pointing to the next Td entry that is higher than the peer's max confirmed Td (nil if it has already confirmed
// the current global head).
type updateStatsEntry struct {
time mclock.AbsTime
td *big.Int
next *updateStatsEntry
}
// updateMaxConfirmedTd updates the block delay statistics of active peers. Whenever a new highest Td is confirmed,
// adds it to the end of a linked list together with the time it has been confirmed. Then checks which peers have
// already confirmed a head with the same or higher Td (which counts as zero block delay) and updates their statistics.
// Those who have not confirmed such a head by now will be updated by a subsequent checkUpdateStats call with a
// positive block delay value.
func (f *lightFetcher) updateMaxConfirmedTd(td *big.Int) {
if f.maxConfirmedTd == nil || td.Cmp(f.maxConfirmedTd) > 0 {
f.maxConfirmedTd = td
newEntry := &updateStatsEntry{
time: mclock.Now(),
td: td,
}
if f.lastUpdateStats != nil {
f.lastUpdateStats.next = newEntry
}
f.lastUpdateStats = newEntry
for p, _ := range f.peers {
f.checkUpdateStats(p, newEntry)
}
}
}
// checkUpdateStats checks those peers who have not confirmed a certain highest Td (or a larger one) by the time it
// has been confirmed by another peer. If they have confirmed such a head by now, their stats are updated with the
// block delay which is (this peer's confirmation time)-(first confirmation time). After blockDelayTimeout has passed,
// the stats are updated with blockDelayTimeout value. In either case, the confirmed or timed out updateStatsEntry
// items are removed from the head of the linked list.
// If a new entry has been added to the global tail, it is passed as a parameter here even though this function
// assumes that it has already been added, so that if the peer's list is empty (all heads confirmed, head is nil),
// it can set the new head to newEntry.
func (f *lightFetcher) checkUpdateStats(p *peer, newEntry *updateStatsEntry) {
now := mclock.Now()
fp := f.peers[p]
if fp == nil {
glog.V(logger.Debug).Infof("checkUpdateStats: unknown peer")
return
}
if newEntry != nil && fp.firstUpdateStats == nil {
fp.firstUpdateStats = newEntry
}
for fp.firstUpdateStats != nil && fp.firstUpdateStats.time <= now-mclock.AbsTime(blockDelayTimeout) {
f.pm.serverPool.adjustBlockDelay(p.poolEntry, blockDelayTimeout)
fp.firstUpdateStats = fp.firstUpdateStats.next
}
if fp.confirmedTd != nil {
for fp.firstUpdateStats != nil && fp.firstUpdateStats.td.Cmp(fp.confirmedTd) <= 0 {
f.pm.serverPool.adjustBlockDelay(p.poolEntry, time.Duration(now-fp.firstUpdateStats.time))
fp.firstUpdateStats = fp.firstUpdateStats.next
}
}
}

117
les/handler.go
View file

@ -22,8 +22,8 @@ import (
"errors"
"fmt"
"math/big"
"net"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
@ -58,7 +58,7 @@ const (
MaxHeaderProofsFetch = 64 // Amount of merkle proofs to be fetched per retrieval request
MaxTxSend = 64 // Amount of transactions to be send per request
disableClientRemovePeer = true
disableClientRemovePeer = false
)
// errIncompatibleConfig is returned if the requested protocols and configs are
@ -88,7 +88,7 @@ type BlockChain interface {
type txPool interface {
// AddTransactions should add the given transactions to the pool.
AddBatch([]*types.Transaction)
AddBatch([]*types.Transaction) error
}
type ProtocolManager struct {
@ -101,10 +101,7 @@ type ProtocolManager struct {
chainDb ethdb.Database
odr *LesOdr
server *LesServer
topicDisc *discv5.Network
lesTopic discv5.Topic
p2pServer *p2p.Server
serverPool *serverPool
downloader *downloader.Downloader
fetcher *lightFetcher
@ -157,13 +154,29 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, network
Version: version,
Length: ProtocolLengths[i],
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
var entry *poolEntry
peer := manager.newPeer(int(version), networkId, p, rw)
if manager.serverPool != nil {
addr := p.RemoteAddr().(*net.TCPAddr)
entry = manager.serverPool.connect(peer, addr.IP, uint16(addr.Port))
if entry == nil {
return fmt.Errorf("unwanted connection")
}
}
peer.poolEntry = entry
select {
case manager.newPeerCh <- peer:
manager.wg.Add(1)
defer manager.wg.Done()
return manager.handle(peer)
err := manager.handle(peer)
if entry != nil {
manager.serverPool.disconnect(entry)
}
return err
case <-manager.quitSync:
if entry != nil {
manager.serverPool.disconnect(entry)
}
return p2p.DiscQuitting
}
},
@ -192,7 +205,6 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, network
manager.downloader = downloader.New(downloader.LightSync, chainDb, manager.eventMux, blockchain.HasHeader, nil, blockchain.GetHeaderByHash,
nil, blockchain.CurrentHeader, nil, nil, nil, blockchain.GetTdByHash,
blockchain.InsertHeaderChain, nil, nil, blockchain.Rollback, removePeer)
manager.fetcher = newLightFetcher(manager)
}
if odr != nil {
@ -222,10 +234,12 @@ func (pm *ProtocolManager) removePeer(id string) {
glog.V(logger.Debug).Infof("LES: unregister peer %v", id)
if pm.lightSync {
pm.downloader.UnregisterPeer(id)
pm.odr.UnregisterPeer(peer)
if pm.txrelay != nil {
pm.txrelay.removePeer(id)
}
if pm.fetcher != nil {
pm.fetcher.removePeer(peer)
}
}
if err := pm.peers.Unregister(id); err != nil {
glog.V(logger.Error).Infoln("Removal failed:", err)
@ -236,54 +250,26 @@ func (pm *ProtocolManager) removePeer(id string) {
}
}
func (pm *ProtocolManager) findServers() {
if pm.p2pServer == nil || pm.topicDisc == nil {
return
}
glog.V(logger.Debug).Infoln("Looking for topic", string(pm.lesTopic))
enodes := make(chan string, 100)
stop := make(chan struct{})
go pm.topicDisc.SearchTopic(pm.lesTopic, stop, enodes)
go func() {
added := make(map[string]bool)
for {
select {
case enode := <-enodes:
if !added[enode] {
glog.V(logger.Info).Infoln("Found LES server:", enode)
added[enode] = true
if node, err := discover.ParseNode(enode); err == nil {
pm.p2pServer.AddPeer(node)
}
}
case <-stop:
return
}
}
}()
select {
case <-time.After(time.Second * 20):
case <-pm.quitSync:
}
close(stop)
}
func (pm *ProtocolManager) Start(srvr *p2p.Server) {
pm.p2pServer = srvr
var topicDisc *discv5.Network
if srvr != nil {
pm.topicDisc = srvr.DiscV5
topicDisc = srvr.DiscV5
}
pm.lesTopic = discv5.Topic("LES@" + common.Bytes2Hex(pm.blockchain.Genesis().Hash().Bytes()[0:8]))
lesTopic := discv5.Topic("LES@" + common.Bytes2Hex(pm.blockchain.Genesis().Hash().Bytes()[0:8]))
if pm.lightSync {
// start sync handler
go pm.findServers()
if srvr != nil { // srvr is nil during testing
pm.serverPool = newServerPool(pm.chainDb, []byte("serverPool/"), srvr, lesTopic, pm.quitSync, &pm.wg)
pm.odr.serverPool = pm.serverPool
pm.fetcher = newLightFetcher(pm)
}
go pm.syncer()
} else {
if pm.topicDisc != nil {
if topicDisc != nil {
go func() {
glog.V(logger.Debug).Infoln("Starting registering topic", string(pm.lesTopic))
pm.topicDisc.RegisterTopic(pm.lesTopic, pm.quitSync)
glog.V(logger.Debug).Infoln("Stopped registering topic", string(pm.lesTopic))
glog.V(logger.Info).Infoln("Starting registering topic", string(lesTopic))
topicDisc.RegisterTopic(lesTopic, pm.quitSync)
glog.V(logger.Info).Infoln("Stopped registering topic", string(lesTopic))
}()
}
go func() {
@ -352,13 +338,13 @@ func (pm *ProtocolManager) handle(p *peer) error {
glog.V(logger.Debug).Infof("LES: register peer %v", p.id)
if pm.lightSync {
requestHeadersByHash := func(origin common.Hash, amount int, skip int, reverse bool) error {
reqID := pm.odr.getNextReqID()
reqID := getNextReqID()
cost := p.GetRequestCost(GetBlockHeadersMsg, amount)
p.fcServer.SendRequest(reqID, cost)
return p.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse)
}
requestHeadersByNumber := func(origin uint64, amount int, skip int, reverse bool) error {
reqID := pm.odr.getNextReqID()
reqID := getNextReqID()
cost := p.GetRequestCost(GetBlockHeadersMsg, amount)
p.fcServer.SendRequest(reqID, cost)
return p.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse)
@ -367,12 +353,21 @@ func (pm *ProtocolManager) handle(p *peer) error {
requestHeadersByHash, requestHeadersByNumber, nil, nil, nil); err != nil {
return err
}
pm.odr.RegisterPeer(p)
if pm.txrelay != nil {
pm.txrelay.addPeer(p)
}
pm.fetcher.notify(p, nil)
p.lock.Lock()
head := p.headInfo
p.lock.Unlock()
if pm.fetcher != nil {
pm.fetcher.addPeer(p)
pm.fetcher.announce(p, head)
}
if p.poolEntry != nil {
pm.serverPool.registered(p.poolEntry)
}
}
stop := make(chan struct{})
@ -454,7 +449,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "%v: %v", msg, err)
}
glog.V(logger.Detail).Infoln("AnnounceMsg:", req.Number, req.Hash, req.Td, req.ReorgDepth)
pm.fetcher.notify(p, &req)
if pm.fetcher != nil {
go pm.fetcher.announce(p, &req)
}
case GetBlockHeadersMsg:
glog.V(logger.Debug).Infof("<=== GetBlockHeadersMsg from peer %v", p.id)
@ -552,8 +549,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
p.fcServer.GotReply(resp.ReqID, resp.BV)
if pm.fetcher.requestedID(resp.ReqID) {
pm.fetcher.deliverHeaders(resp.ReqID, resp.Headers)
if pm.fetcher != nil && pm.fetcher.requestedID(resp.ReqID) {
pm.fetcher.deliverHeaders(p, resp.ReqID, resp.Headers)
} else {
err := pm.downloader.DeliverHeaders(p.id, resp.Headers)
if err != nil {
@ -879,7 +876,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if reqCnt > maxReqs || reqCnt > MaxTxSend {
return errResp(ErrRequestRejected, "")
}
pm.txpool.AddBatch(txs)
if err := pm.txpool.AddBatch(txs); err != nil {
return errResp(ErrUnexpectedResponse, "msg: %v", err)
}
_, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)

11
les/helper_test.go
View file

@ -25,6 +25,7 @@ import (
"math/big"
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
@ -334,3 +335,13 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu
func (p *testPeer) close() {
p.app.Close()
}
type testServerPool peer
func (p *testServerPool) selectPeer(func(*peer) (bool, uint64)) *peer {
return (*peer)(p)
}
func (p *testServerPool) adjustResponseTime(*poolEntry, time.Duration, bool) {
}

73
les/odr.go
View file

@ -17,6 +17,8 @@
package les
import (
"crypto/rand"
"encoding/binary"
"sync"
"time"
@ -37,6 +39,11 @@ var (
// peerDropFn is a callback type for dropping a peer detected as malicious.
type peerDropFn func(id string)
type odrPeerSelector interface {
selectPeer(func(*peer) (bool, uint64)) *peer
adjustResponseTime(*poolEntry, time.Duration, bool)
}
type LesOdr struct {
light.OdrBackend
db ethdb.Database
@ -44,15 +51,13 @@ type LesOdr struct {
removePeer peerDropFn
mlock, clock sync.Mutex
sentReqs map[uint64]*sentReq
peers *odrPeerSet
lastReqID uint64
serverPool odrPeerSelector
}
func NewLesOdr(db ethdb.Database) *LesOdr {
return &LesOdr{
db: db,
stop: make(chan struct{}),
peers: newOdrPeerSet(),
sentReqs: make(map[uint64]*sentReq),
}
}
@ -77,16 +82,6 @@ type sentReq struct {
answered chan struct{} // closed and set to nil when any peer answers it
}
// RegisterPeer registers a new LES peer to the ODR capable peer set
func (self *LesOdr) RegisterPeer(p *peer) error {
return self.peers.register(p)
}
// UnregisterPeer removes a peer from the ODR capable peer set
func (self *LesOdr) UnregisterPeer(p *peer) {
self.peers.unregister(p)
}
const (
MsgBlockBodies = iota
MsgCode
@ -142,29 +137,26 @@ func (self *LesOdr) requestPeer(req *sentReq, peer *peer, delivered, timeout cha
select {
case <-delivered:
servTime := uint64(mclock.Now() - stime)
self.peers.updateTimeout(peer, false)
self.peers.updateServTime(peer, servTime)
if self.serverPool != nil {
self.serverPool.adjustResponseTime(peer.poolEntry, time.Duration(mclock.Now()-stime), false)
}
return
case <-time.After(softRequestTimeout):
close(timeout)
if self.peers.updateTimeout(peer, true) {
self.removePeer(peer.id)
}
case <-self.stop:
return
}
select {
case <-delivered:
servTime := uint64(mclock.Now() - stime)
self.peers.updateServTime(peer, servTime)
return
case <-time.After(hardRequestTimeout):
self.removePeer(peer.id)
go self.removePeer(peer.id)
case <-self.stop:
return
}
if self.serverPool != nil {
self.serverPool.adjustResponseTime(peer.poolEntry, time.Duration(mclock.Now()-stime), true)
}
}
// networkRequest sends a request to known peers until an answer is received
@ -176,7 +168,7 @@ func (self *LesOdr) networkRequest(ctx context.Context, lreq LesOdrRequest) erro
sentTo: make(map[*peer]chan struct{}),
answered: answered, // reply delivered by any peer
}
reqID := self.getNextReqID()
reqID := getNextReqID()
self.mlock.Lock()
self.sentReqs[reqID] = req
self.mlock.Unlock()
@ -193,7 +185,16 @@ func (self *LesOdr) networkRequest(ctx context.Context, lreq LesOdrRequest) erro
exclude := make(map[*peer]struct{})
for {
if peer := self.peers.bestPeer(lreq, exclude); peer == nil {
var p *peer
if self.serverPool != nil {
p = self.serverPool.selectPeer(func(p *peer) (bool, uint64) {
if !lreq.CanSend(p) {
return false, 0
}
return true, p.fcServer.CanSend(lreq.GetCost(p))
})
}
if p == nil {
select {
case <-ctx.Done():
return ctx.Err()
@ -202,17 +203,17 @@ func (self *LesOdr) networkRequest(ctx context.Context, lreq LesOdrRequest) erro
case <-time.After(retryPeers):
}
} else {
exclude[peer] = struct{}{}
exclude[p] = struct{}{}
delivered := make(chan struct{})
timeout := make(chan struct{})
req.lock.Lock()
req.sentTo[peer] = delivered
req.sentTo[p] = delivered
req.lock.Unlock()
reqWg.Add(1)
cost := lreq.GetCost(peer)
peer.fcServer.SendRequest(reqID, cost)
go self.requestPeer(req, peer, delivered, timeout, reqWg)
lreq.Request(reqID, peer)
cost := lreq.GetCost(p)
p.fcServer.SendRequest(reqID, cost)
go self.requestPeer(req, p, delivered, timeout, reqWg)
lreq.Request(reqID, p)
select {
case <-ctx.Done():
@ -239,10 +240,8 @@ func (self *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err err
return
}
func (self *LesOdr) getNextReqID() uint64 {
self.clock.Lock()
defer self.clock.Unlock()
self.lastReqID++
return self.lastReqID
func getNextReqID() uint64 {
var rnd [8]byte
rand.Read(rnd[:])
return binary.BigEndian.Uint64(rnd[:])
}

120
les/odr_peerset.go
View file

@ -1,120 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package les
import (
"sync"
)
const dropTimeoutRatio = 20
type odrPeerInfo struct {
reqTimeSum, reqTimeCnt, reqCnt, timeoutCnt uint64
}
// odrPeerSet represents the collection of active peer participating in the block
// download procedure.
type odrPeerSet struct {
peers map[*peer]*odrPeerInfo
lock sync.RWMutex
}
// newPeerSet creates a new peer set top track the active download sources.
func newOdrPeerSet() *odrPeerSet {
return &odrPeerSet{
peers: make(map[*peer]*odrPeerInfo),
}
}
// Register injects a new peer into the working set, or returns an error if the
// peer is already known.
func (ps *odrPeerSet) register(p *peer) error {
ps.lock.Lock()
defer ps.lock.Unlock()
if _, ok := ps.peers[p]; ok {
return errAlreadyRegistered
}
ps.peers[p] = &odrPeerInfo{}
return nil
}
// Unregister removes a remote peer from the active set, disabling any further
// actions to/from that particular entity.
func (ps *odrPeerSet) unregister(p *peer) error {
ps.lock.Lock()
defer ps.lock.Unlock()
if _, ok := ps.peers[p]; !ok {
return errNotRegistered
}
delete(ps.peers, p)
return nil
}
func (ps *odrPeerSet) peerPriority(p *peer, info *odrPeerInfo, req LesOdrRequest) uint64 {
tm := p.fcServer.CanSend(req.GetCost(p))
if info.reqTimeCnt > 0 {
tm += info.reqTimeSum / info.reqTimeCnt
}
return tm
}
func (ps *odrPeerSet) bestPeer(req LesOdrRequest, exclude map[*peer]struct{}) *peer {
var best *peer
var bpv uint64
ps.lock.Lock()
defer ps.lock.Unlock()
for p, info := range ps.peers {
if _, ok := exclude[p]; !ok {
pv := ps.peerPriority(p, info, req)
if best == nil || pv < bpv {
best = p
bpv = pv
}
}
}
return best
}
func (ps *odrPeerSet) updateTimeout(p *peer, timeout bool) (drop bool) {
ps.lock.Lock()
defer ps.lock.Unlock()
if info, ok := ps.peers[p]; ok {
info.reqCnt++
if timeout {
// check ratio before increase to allow an extra timeout
if info.timeoutCnt*dropTimeoutRatio >= info.reqCnt {
return true
}
info.timeoutCnt++
}
}
return false
}
func (ps *odrPeerSet) updateServTime(p *peer, servTime uint64) {
ps.lock.Lock()
defer ps.lock.Unlock()
if info, ok := ps.peers[p]; ok {
info.reqTimeSum += servTime
info.reqTimeCnt++
}
}

29
les/odr_requests.go
View file

@ -36,6 +36,7 @@ import (
type LesOdrRequest interface {
GetCost(*peer) uint64
CanSend(*peer) bool
Request(uint64, *peer) error
Valid(ethdb.Database, *Msg) bool // if true, keeps the retrieved object
}
@ -66,6 +67,11 @@ func (self *BlockRequest) GetCost(peer *peer) uint64 {
return peer.GetRequestCost(GetBlockBodiesMsg, 1)
}
// CanSend tells if a certain peer is suitable for serving the given request
func (self *BlockRequest) CanSend(peer *peer) bool {
return peer.HasBlock(self.Hash, self.Number)
}
// Request sends an ODR request to the LES network (implementation of LesOdrRequest)
func (self *BlockRequest) Request(reqID uint64, peer *peer) error {
glog.V(logger.Debug).Infof("ODR: requesting body of block %08x from peer %v", self.Hash[:4], peer.id)
@ -121,6 +127,11 @@ func (self *ReceiptsRequest) GetCost(peer *peer) uint64 {
return peer.GetRequestCost(GetReceiptsMsg, 1)
}
// CanSend tells if a certain peer is suitable for serving the given request
func (self *ReceiptsRequest) CanSend(peer *peer) bool {
return peer.HasBlock(self.Hash, self.Number)
}
// Request sends an ODR request to the LES network (implementation of LesOdrRequest)
func (self *ReceiptsRequest) Request(reqID uint64, peer *peer) error {
glog.V(logger.Debug).Infof("ODR: requesting receipts for block %08x from peer %v", self.Hash[:4], peer.id)
@ -171,6 +182,11 @@ func (self *TrieRequest) GetCost(peer *peer) uint64 {
return peer.GetRequestCost(GetProofsMsg, 1)
}
// CanSend tells if a certain peer is suitable for serving the given request
func (self *TrieRequest) CanSend(peer *peer) bool {
return peer.HasBlock(self.Id.BlockHash, self.Id.BlockNumber)
}
// Request sends an ODR request to the LES network (implementation of LesOdrRequest)
func (self *TrieRequest) Request(reqID uint64, peer *peer) error {
glog.V(logger.Debug).Infof("ODR: requesting trie root %08x key %08x from peer %v", self.Id.Root[:4], self.Key[:4], peer.id)
@ -221,6 +237,11 @@ func (self *CodeRequest) GetCost(peer *peer) uint64 {
return peer.GetRequestCost(GetCodeMsg, 1)
}
// CanSend tells if a certain peer is suitable for serving the given request
func (self *CodeRequest) CanSend(peer *peer) bool {
return peer.HasBlock(self.Id.BlockHash, self.Id.BlockNumber)
}
// Request sends an ODR request to the LES network (implementation of LesOdrRequest)
func (self *CodeRequest) Request(reqID uint64, peer *peer) error {
glog.V(logger.Debug).Infof("ODR: requesting node data for hash %08x from peer %v", self.Hash[:4], peer.id)
@ -274,6 +295,14 @@ func (self *ChtRequest) GetCost(peer *peer) uint64 {
return peer.GetRequestCost(GetHeaderProofsMsg, 1)
}
// CanSend tells if a certain peer is suitable for serving the given request
func (self *ChtRequest) CanSend(peer *peer) bool {
peer.lock.RLock()
defer peer.lock.RUnlock()
return self.ChtNum <= (peer.headInfo.Number-light.ChtConfirmations)/light.ChtFrequency
}
// Request sends an ODR request to the LES network (implementation of LesOdrRequest)
func (self *ChtRequest) Request(reqID uint64, peer *peer) error {
glog.V(logger.Debug).Infof("ODR: requesting CHT #%d block #%d from peer %v", self.ChtNum, self.BlockNum, peer.id)

8
les/odr_test.go
View file

@ -160,6 +160,8 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) {
pm, db, odr := newTestProtocolManagerMust(t, false, 4, testChainGen)
lpm, ldb, odr := newTestProtocolManagerMust(t, true, 0, nil)
_, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm)
pool := (*testServerPool)(lpeer)
odr.serverPool = pool
select {
case <-time.After(time.Millisecond * 100):
case err := <-err1:
@ -188,13 +190,13 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) {
}
// temporarily remove peer to test odr fails
odr.UnregisterPeer(lpeer)
odr.serverPool = nil
// expect retrievals to fail (except genesis block) without a les peer
test(expFail)
odr.RegisterPeer(lpeer)
odr.serverPool = pool
// expect all retrievals to pass
test(5)
odr.UnregisterPeer(lpeer)
odr.serverPool = nil
// still expect all retrievals to pass, now data should be cached locally
test(5)
}

84
les/peer.go
View file

@ -51,12 +51,14 @@ type peer struct {
id string
firstHeadInfo, headInfo *announceData
headInfoLen int
lock sync.RWMutex
headInfo *announceData
lock sync.RWMutex
announceChn chan announceData
poolEntry *poolEntry
hasBlock func(common.Hash, uint64) bool
fcClient *flowcontrol.ClientNode // nil if the peer is server only
fcServer *flowcontrol.ServerNode // nil if the peer is client only
fcServerParams *flowcontrol.ServerParams
@ -109,67 +111,6 @@ func (p *peer) headBlockInfo() blockInfo {
return blockInfo{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}
}
func (p *peer) addNotify(announce *announceData) bool {
p.lock.Lock()
defer p.lock.Unlock()
if announce.Td.Cmp(p.headInfo.Td) < 1 {
return false
}
if p.headInfoLen >= maxHeadInfoLen {
//return false
p.firstHeadInfo = p.firstHeadInfo.next
p.headInfoLen--
}
if announce.haveHeaders == 0 {
hh := p.headInfo.Number - announce.ReorgDepth
if p.headInfo.haveHeaders < hh {
hh = p.headInfo.haveHeaders
}
announce.haveHeaders = hh
}
p.headInfo.next = announce
p.headInfo = announce
p.headInfoLen++
return true
}
func (p *peer) gotHeader(hash common.Hash, number uint64, td *big.Int) bool {
h := p.firstHeadInfo
ptr := 0
for h != nil {
if h.Hash == hash {
if h.Number != number || h.Td.Cmp(td) != 0 {
return false
}
h.headKnown = true
h.haveHeaders = h.Number
p.firstHeadInfo = h
p.headInfoLen -= ptr
last := h
h = h.next
// propagate haveHeaders through the chain
for h != nil {
hh := last.Number - h.ReorgDepth
if last.haveHeaders < hh {
hh = last.haveHeaders
}
if hh > h.haveHeaders {
h.haveHeaders = hh
} else {
return true
}
last = h
h = h.next
}
return true
}
h = h.next
ptr++
}
return true
}
// Td retrieves the current total difficulty of a peer.
func (p *peer) Td() *big.Int {
p.lock.RLock()
@ -195,6 +136,9 @@ func sendResponse(w p2p.MsgWriter, msgcode, reqID, bv uint64, data interface{})
}
func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 {
p.lock.RLock()
defer p.lock.RUnlock()
cost := p.fcCosts[msgcode].baseCost + p.fcCosts[msgcode].reqCost*uint64(amount)
if cost > p.fcServerParams.BufLimit {
cost = p.fcServerParams.BufLimit
@ -202,6 +146,14 @@ func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 {
return cost
}
// HasBlock checks if the peer has a given block
func (p *peer) HasBlock(hash common.Hash, number uint64) bool {
p.lock.RLock()
hashBlock := p.hasBlock
p.lock.RUnlock()
return hashBlock != nil && hashBlock(hash, number)
}
// SendAnnounce announces the availability of a number of blocks through
// a hash notification.
func (p *peer) SendAnnounce(request announceData) error {
@ -453,9 +405,7 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis
p.fcCosts = MRC.decode()
}
p.firstHeadInfo = &announceData{Td: rTd, Hash: rHash, Number: rNum}
p.headInfo = p.firstHeadInfo
p.headInfoLen = 1
p.headInfo = &announceData{Td: rTd, Hash: rHash, Number: rNum}
return nil
}

173
les/randselect.go Normal file
View file

@ -0,0 +1,173 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package les implements the Light Ethereum Subprotocol.
package les
import (
"math/rand"
)
// wrsItem interface should be implemented by any entries that are to be selected from
// a weightedRandomSelect set. Note that recalculating monotonously decreasing item
// weights on-demand (without constantly calling update) is allowed
type wrsItem interface {
Weight() int64
}
// weightedRandomSelect is capable of weighted random selection from a set of items
type weightedRandomSelect struct {
root *wrsNode
idx map[wrsItem]int
}
// newWeightedRandomSelect returns a new weightedRandomSelect structure
func newWeightedRandomSelect() *weightedRandomSelect {
return &weightedRandomSelect{root: &wrsNode{maxItems: wrsBranches}, idx: make(map[wrsItem]int)}
}
// update updates an item's weight, adds it if it was non-existent or removes it if
// the new weight is zero. Note that explicitly updating decreasing weights is not necessary.
func (w *weightedRandomSelect) update(item wrsItem) {
w.setWeight(item, item.Weight())
}
// remove removes an item from the set
func (w *weightedRandomSelect) remove(item wrsItem) {
w.setWeight(item, 0)
}
// setWeight sets an item's weight to a specific value (removes it if zero)
func (w *weightedRandomSelect) setWeight(item wrsItem, weight int64) {
idx, ok := w.idx[item]
if ok {
w.root.setWeight(idx, weight)
if weight == 0 {
delete(w.idx, item)
}
} else {
if weight != 0 {
if w.root.itemCnt == w.root.maxItems {
// add a new level
newRoot := &wrsNode{sumWeight: w.root.sumWeight, itemCnt: w.root.itemCnt, level: w.root.level + 1, maxItems: w.root.maxItems * wrsBranches}
newRoot.items[0] = w.root
newRoot.weights[0] = w.root.sumWeight
w.root = newRoot
}
w.idx[item] = w.root.insert(item, weight)
}
}
}
// choose randomly selects an item from the set, with a chance proportional to its
// current weight. If the weight of the chosen element has been decreased since the
// last stored value, returns it with a newWeight/oldWeight chance, otherwise just
// updates its weight and selects another one
func (w *weightedRandomSelect) choose() wrsItem {
for {
if w.root.sumWeight == 0 {
return nil
}
val := rand.Int63n(w.root.sumWeight)
choice, lastWeight := w.root.choose(val)
weight := choice.Weight()
if weight != lastWeight {
w.setWeight(choice, weight)
}
if weight >= lastWeight || rand.Int63n(lastWeight) < weight {
return choice
}
}
}
const wrsBranches = 8 // max number of branches in the wrsNode tree
// wrsNode is a node of a tree structure that can store wrsItems or further wrsNodes.
type wrsNode struct {
items [wrsBranches]interface{}
weights [wrsBranches]int64
sumWeight int64
level, itemCnt, maxItems int
}
// insert recursively inserts a new item to the tree and returns the item index
func (n *wrsNode) insert(item wrsItem, weight int64) int {
branch := 0
for n.items[branch] != nil && (n.level == 0 || n.items[branch].(*wrsNode).itemCnt == n.items[branch].(*wrsNode).maxItems) {
branch++
if branch == wrsBranches {
panic(nil)
}
}
n.itemCnt++
n.sumWeight += weight
n.weights[branch] += weight
if n.level == 0 {
n.items[branch] = item
return branch
} else {
var subNode *wrsNode
if n.items[branch] == nil {
subNode = &wrsNode{maxItems: n.maxItems / wrsBranches, level: n.level - 1}
n.items[branch] = subNode
} else {
subNode = n.items[branch].(*wrsNode)
}
subIdx := subNode.insert(item, weight)
return subNode.maxItems*branch + subIdx
}
}
// setWeight updates the weight of a certain item (which should exist) and returns
// the change of the last weight value stored in the tree
func (n *wrsNode) setWeight(idx int, weight int64) int64 {
if n.level == 0 {
oldWeight := n.weights[idx]
n.weights[idx] = weight
diff := weight - oldWeight
n.sumWeight += diff
if weight == 0 {
n.items[idx] = nil
n.itemCnt--
}
return diff
}
branchItems := n.maxItems / wrsBranches
branch := idx / branchItems
diff := n.items[branch].(*wrsNode).setWeight(idx-branch*branchItems, weight)
n.weights[branch] += diff
n.sumWeight += diff
if weight == 0 {
n.itemCnt--
}
return diff
}
// choose recursively selects an item from the tree and returns it along with its weight
func (n *wrsNode) choose(val int64) (wrsItem, int64) {
for i, w := range n.weights {
if val < w {
if n.level == 0 {
return n.items[i].(wrsItem), n.weights[i]
} else {
return n.items[i].(*wrsNode).choose(val)
}
} else {
val -= w
}
}
panic(nil)
}

67
les/randselect_test.go Normal file
View file

@ -0,0 +1,67 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package les
import (
"math/rand"
"testing"
)
type testWrsItem struct {
idx int
widx *int
}
func (t *testWrsItem) Weight() int64 {
w := *t.widx
if w == -1 || w == t.idx {
return int64(t.idx + 1)
}
return 0
}
func TestWeightedRandomSelect(t *testing.T) {
testFn := func(cnt int) {
s := newWeightedRandomSelect()
w := -1
list := make([]testWrsItem, cnt)
for i, _ := range list {
list[i] = testWrsItem{idx: i, widx: &w}
s.update(&list[i])
}
w = rand.Intn(cnt)
c := s.choose()
if c == nil {
t.Errorf("expected item, got nil")
} else {
if c.(*testWrsItem).idx != w {
t.Errorf("expected another item")
}
}
w = -2
if s.choose() != nil {
t.Errorf("expected nil, got item")
}
}
testFn(1)
testFn(10)
testFn(100)
testFn(1000)
testFn(10000)
testFn(100000)
testFn(1000000)
}

7
les/request_test.go
View file

@ -71,6 +71,8 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) {
pm, db, _ := newTestProtocolManagerMust(t, false, 4, testChainGen)
lpm, ldb, odr := newTestProtocolManagerMust(t, true, 0, nil)
_, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm)
pool := (*testServerPool)(lpeer)
odr.serverPool = pool
select {
case <-time.After(time.Millisecond * 100):
case err := <-err1:
@ -100,11 +102,10 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) {
}
// temporarily remove peer to test odr fails
odr.UnregisterPeer(lpeer)
odr.serverPool = nil
// expect retrievals to fail (except genesis block) without a les peer
test(0)
odr.RegisterPeer(lpeer)
odr.serverPool = pool
// expect all retrievals to pass
test(5)
odr.UnregisterPeer(lpeer)
}

37
les/server.go
View file

@ -42,6 +42,9 @@ type LesServer struct {
fcManager *flowcontrol.ClientManager // nil if our node is client only
fcCostStats *requestCostStats
defParams *flowcontrol.ServerParams
srvr *p2p.Server
synced, stopped bool
lock sync.Mutex
}
func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) {
@ -67,12 +70,35 @@ func (s *LesServer) Protocols() []p2p.Protocol {
return s.protocolManager.SubProtocols
}
// Start only starts the actual service if the ETH protocol has already been synced,
// otherwise it will be started by Synced()
func (s *LesServer) Start(srvr *p2p.Server) {
s.protocolManager.Start(srvr)
s.lock.Lock()
defer s.lock.Unlock()
s.srvr = srvr
if s.synced {
s.protocolManager.Start(s.srvr)
}
}
// Synced notifies the server that the ETH protocol has been synced and LES service can be started
func (s *LesServer) Synced() {
s.lock.Lock()
defer s.lock.Unlock()
s.synced = true
if s.srvr != nil && !s.stopped {
s.protocolManager.Start(s.srvr)
}
}
// Stop stops the LES service
func (s *LesServer) Stop() {
s.lock.Lock()
defer s.lock.Unlock()
s.stopped = true
s.fcCostStats.store()
s.fcManager.Stop()
go func() {
@ -323,9 +349,8 @@ func (pm *ProtocolManager) blockLoop() {
}
var (
lastChtKey = []byte("LastChtNumber") // chtNum (uint64 big endian)
chtPrefix = []byte("cht") // chtPrefix + chtNum (uint64 big endian) -> trie root hash
chtConfirmations = light.ChtFrequency / 2
lastChtKey = []byte("LastChtNumber") // chtNum (uint64 big endian)
chtPrefix = []byte("cht") // chtPrefix + chtNum (uint64 big endian) -> trie root hash
)
func getChtRoot(db ethdb.Database, num uint64) common.Hash {
@ -346,8 +371,8 @@ func makeCht(db ethdb.Database) bool {
headNum := core.GetBlockNumber(db, headHash)
var newChtNum uint64
if headNum > chtConfirmations {
newChtNum = (headNum - chtConfirmations) / light.ChtFrequency
if headNum > light.ChtConfirmations {
newChtNum = (headNum - light.ChtConfirmations) / light.ChtFrequency
}
var lastChtNum uint64

766
les/serverpool.go Normal file
View file

@ -0,0 +1,766 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package les implements the Light Ethereum Subprotocol.
package les
import (
"io"
"math"
"math/rand"
"net"
"strconv"
"sync"
"time"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/discv5"
"github.com/ethereum/go-ethereum/rlp"
)
const (
// After a connection has been ended or timed out, there is a waiting period
// before it can be selected for connection again.
// waiting period = base delay * (1 + random(1))
// base delay = shortRetryDelay for the first shortRetryCnt times after a
// successful connection, after that longRetryDelay is applied
shortRetryCnt = 5
shortRetryDelay = time.Second * 5
longRetryDelay = time.Minute * 10
// maxNewEntries is the maximum number of newly discovered (never connected) nodes.
// If the limit is reached, the least recently discovered one is thrown out.
maxNewEntries = 1000
// maxKnownEntries is the maximum number of known (already connected) nodes.
// If the limit is reached, the least recently connected one is thrown out.
// (not that unlike new entries, known entries are persistent)
maxKnownEntries = 1000
// target for simultaneously connected servers
targetServerCount = 5
// target for servers selected from the known table
// (we leave room for trying new ones if there is any)
targetKnownSelect = 3
// after dialTimeout, consider the server unavailable and adjust statistics
dialTimeout = time.Second * 30
// targetConnTime is the minimum expected connection duration before a server
// drops a client without any specific reason
targetConnTime = time.Minute * 10
// new entry selection weight calculation based on most recent discovery time:
// unity until discoverExpireStart, then exponential decay with discoverExpireConst
discoverExpireStart = time.Minute * 20
discoverExpireConst = time.Minute * 20
// known entry selection weight is dropped by a factor of exp(-failDropLn) after
// each unsuccessful connection (restored after a successful one)
failDropLn = 0.1
// known node connection success and quality statistics have a long term average
// and a short term value which is adjusted exponentially with a factor of
// pstatRecentAdjust with each dial/connection and also returned exponentially
// to the average with the time constant pstatReturnToMeanTC
pstatRecentAdjust = 0.1
pstatReturnToMeanTC = time.Hour
// node address selection weight is dropped by a factor of exp(-addrFailDropLn) after
// each unsuccessful connection (restored after a successful one)
addrFailDropLn = math.Ln2
// responseScoreTC and delayScoreTC are exponential decay time constants for
// calculating selection chances from response times and block delay times
responseScoreTC = time.Millisecond * 100
delayScoreTC = time.Second * 5
timeoutPow = 10
// peerSelectMinWeight is added to calculated weights at request peer selection
// to give poorly performing peers a little chance of coming back
peerSelectMinWeight = 0.005
// initStatsWeight is used to initialize previously unknown peers with good
// statistics to give a chance to prove themselves
initStatsWeight = 1
)
// serverPool implements a pool for storing and selecting newly discovered and already
// known light server nodes. It received discovered nodes, stores statistics about
// known nodes and takes care of always having enough good quality servers connected.
type serverPool struct {
db ethdb.Database
dbKey []byte
server *p2p.Server
quit chan struct{}
wg *sync.WaitGroup
connWg sync.WaitGroup
discSetPeriod chan time.Duration
discNodes chan *discv5.Node
discLookups chan bool
entries map[discover.NodeID]*poolEntry
lock sync.Mutex
timeout, enableRetry chan *poolEntry
adjustStats chan poolStatAdjust
knownQueue, newQueue poolEntryQueue
knownSelect, newSelect *weightedRandomSelect
knownSelected, newSelected int
fastDiscover bool
}
// newServerPool creates a new serverPool instance
func newServerPool(db ethdb.Database, dbPrefix []byte, server *p2p.Server, topic discv5.Topic, quit chan struct{}, wg *sync.WaitGroup) *serverPool {
pool := &serverPool{
db: db,
dbKey: append(dbPrefix, []byte(topic)...),
server: server,
quit: quit,
wg: wg,
entries: make(map[discover.NodeID]*poolEntry),
timeout: make(chan *poolEntry, 1),
adjustStats: make(chan poolStatAdjust, 100),
enableRetry: make(chan *poolEntry, 1),
knownSelect: newWeightedRandomSelect(),
newSelect: newWeightedRandomSelect(),
fastDiscover: true,
}
pool.knownQueue = newPoolEntryQueue(maxKnownEntries, pool.removeEntry)
pool.newQueue = newPoolEntryQueue(maxNewEntries, pool.removeEntry)
wg.Add(1)
pool.loadNodes()
pool.checkDial()
if pool.server.DiscV5 != nil {
pool.discSetPeriod = make(chan time.Duration, 1)
pool.discNodes = make(chan *discv5.Node, 100)
pool.discLookups = make(chan bool, 100)
go pool.server.DiscV5.SearchTopic(topic, pool.discSetPeriod, pool.discNodes, pool.discLookups)
}
go pool.eventLoop()
return pool
}
// connect should be called upon any incoming connection. If the connection has been
// dialed by the server pool recently, the appropriate pool entry is returned.
// Otherwise, the connection should be rejected.
// Note that whenever a connection has been accepted and a pool entry has been returned,
// disconnect should also always be called.
func (pool *serverPool) connect(p *peer, ip net.IP, port uint16) *poolEntry {
pool.lock.Lock()
defer pool.lock.Unlock()
entry := pool.entries[p.ID()]
if entry == nil {
return nil
}
glog.V(logger.Debug).Infof("connecting to %v, state: %v", p.id, entry.state)
if entry.state != psDialed {
return nil
}
pool.connWg.Add(1)
entry.peer = p
entry.state = psConnected
addr := &poolEntryAddress{
ip: ip,
port: port,
lastSeen: mclock.Now(),
}
entry.lastConnected = addr
entry.addr = make(map[string]*poolEntryAddress)
entry.addr[addr.strKey()] = addr
entry.addrSelect = *newWeightedRandomSelect()
entry.addrSelect.update(addr)
return entry
}
// registered should be called after a successful handshake
func (pool *serverPool) registered(entry *poolEntry) {
glog.V(logger.Debug).Infof("registered %v", entry.id.String())
pool.lock.Lock()
defer pool.lock.Unlock()
entry.state = psRegistered
entry.regTime = mclock.Now()
if !entry.known {
pool.newQueue.remove(entry)
entry.known = true
}
pool.knownQueue.setLatest(entry)
entry.shortRetry = shortRetryCnt
}
// disconnect should be called when ending a connection. Service quality statistics
// can be updated optionally (not updated if no registration happened, in this case
// only connection statistics are updated, just like in case of timeout)
func (pool *serverPool) disconnect(entry *poolEntry) {
glog.V(logger.Debug).Infof("disconnected %v", entry.id.String())
pool.lock.Lock()
defer pool.lock.Unlock()
if entry.state == psRegistered {
connTime := mclock.Now() - entry.regTime
connAdjust := float64(connTime) / float64(targetConnTime)
if connAdjust > 1 {
connAdjust = 1
}
stopped := false
select {
case <-pool.quit:
stopped = true
default:
}
if stopped {
entry.connectStats.add(1, connAdjust)
} else {
entry.connectStats.add(connAdjust, 1)
}
}
entry.state = psNotConnected
if entry.knownSelected {
pool.knownSelected--
} else {
pool.newSelected--
}
pool.setRetryDial(entry)
pool.connWg.Done()
}
const (
pseBlockDelay = iota
pseResponseTime
pseResponseTimeout
)
// poolStatAdjust records are sent to adjust peer block delay/response time statistics
type poolStatAdjust struct {
adjustType int
entry *poolEntry
time time.Duration
}
// adjustBlockDelay adjusts the block announce delay statistics of a node
func (pool *serverPool) adjustBlockDelay(entry *poolEntry, time time.Duration) {
pool.adjustStats <- poolStatAdjust{pseBlockDelay, entry, time}
}
// adjustResponseTime adjusts the request response time statistics of a node
func (pool *serverPool) adjustResponseTime(entry *poolEntry, time time.Duration, timeout bool) {
if timeout {
pool.adjustStats <- poolStatAdjust{pseResponseTimeout, entry, time}
} else {
pool.adjustStats <- poolStatAdjust{pseResponseTime, entry, time}
}
}
type selectPeerItem struct {
peer *peer
weight int64
}
func (sp selectPeerItem) Weight() int64 {
return sp.weight
}
// selectPeer selects a suitable peer for a request
func (pool *serverPool) selectPeer(canSend func(*peer) (bool, uint64)) *peer {
pool.lock.Lock()
defer pool.lock.Unlock()
sel := newWeightedRandomSelect()
for _, entry := range pool.entries {
if entry.state == psRegistered {
p := entry.peer
ok, cost := canSend(p)
if ok {
w := int64(1000000000 * (peerSelectMinWeight + math.Exp(-(entry.responseStats.recentAvg()+float64(cost))/float64(responseScoreTC))*math.Pow((1-entry.timeoutStats.recentAvg()), timeoutPow)))
sel.update(selectPeerItem{peer: p, weight: w})
}
}
}
choice := sel.choose()
if choice == nil {
return nil
}
return choice.(selectPeerItem).peer
}
// eventLoop handles pool events and mutex locking for all internal functions
func (pool *serverPool) eventLoop() {
lookupCnt := 0
var convTime mclock.AbsTime
pool.discSetPeriod <- time.Millisecond * 100
for {
select {
case entry := <-pool.timeout:
pool.lock.Lock()
if !entry.removed {
pool.checkDialTimeout(entry)
}
pool.lock.Unlock()
case entry := <-pool.enableRetry:
pool.lock.Lock()
if !entry.removed {
entry.delayedRetry = false
pool.updateCheckDial(entry)
}
pool.lock.Unlock()
case adj := <-pool.adjustStats:
pool.lock.Lock()
switch adj.adjustType {
case pseBlockDelay:
adj.entry.delayStats.add(float64(adj.time), 1)
case pseResponseTime:
adj.entry.responseStats.add(float64(adj.time), 1)
adj.entry.timeoutStats.add(0, 1)
case pseResponseTimeout:
adj.entry.timeoutStats.add(1, 1)
}
pool.lock.Unlock()
case node := <-pool.discNodes:
pool.lock.Lock()
now := mclock.Now()
id := discover.NodeID(node.ID)
entry := pool.entries[id]
if entry == nil {
glog.V(logger.Debug).Infof("discovered %v", node.String())
entry = &poolEntry{
id: id,
addr: make(map[string]*poolEntryAddress),
addrSelect: *newWeightedRandomSelect(),
shortRetry: shortRetryCnt,
}
pool.entries[id] = entry
// initialize previously unknown peers with good statistics to give a chance to prove themselves
entry.connectStats.add(1, initStatsWeight)
entry.delayStats.add(0, initStatsWeight)
entry.responseStats.add(0, initStatsWeight)
entry.timeoutStats.add(0, initStatsWeight)
}
entry.lastDiscovered = now
addr := &poolEntryAddress{
ip: node.IP,
port: node.TCP,
}
if a, ok := entry.addr[addr.strKey()]; ok {
addr = a
} else {
entry.addr[addr.strKey()] = addr
}
addr.lastSeen = now
entry.addrSelect.update(addr)
if !entry.known {
pool.newQueue.setLatest(entry)
}
pool.updateCheckDial(entry)
pool.lock.Unlock()
case conv := <-pool.discLookups:
if conv {
if lookupCnt == 0 {
convTime = mclock.Now()
}
lookupCnt++
if pool.fastDiscover && (lookupCnt == 50 || time.Duration(mclock.Now()-convTime) > time.Minute) {
pool.fastDiscover = false
pool.discSetPeriod <- time.Minute
}
}
case <-pool.quit:
close(pool.discSetPeriod)
pool.connWg.Wait()
pool.saveNodes()
pool.wg.Done()
return
}
}
}
// loadNodes loads known nodes and their statistics from the database
func (pool *serverPool) loadNodes() {
enc, err := pool.db.Get(pool.dbKey)
if err != nil {
return
}
var list []*poolEntry
err = rlp.DecodeBytes(enc, &list)
if err != nil {
glog.V(logger.Debug).Infof("node list decode error: %v", err)
return
}
for _, e := range list {
glog.V(logger.Debug).Infof("loaded server stats %016x fails: %v connStats: %v / %v delayStats: %v / %v responseStats: %v / %v timeoutStats: %v / %v", e.id[0:8], e.lastConnected.fails, e.connectStats.avg, e.connectStats.weight, time.Duration(e.delayStats.avg), e.delayStats.weight, time.Duration(e.responseStats.avg), e.responseStats.weight, e.timeoutStats.avg, e.timeoutStats.weight)
pool.entries[e.id] = e
pool.knownQueue.setLatest(e)
pool.knownSelect.update((*knownEntry)(e))
}
}
// saveNodes saves known nodes and their statistics into the database. Nodes are
// ordered from least to most recently connected.
func (pool *serverPool) saveNodes() {
list := make([]*poolEntry, len(pool.knownQueue.queue))
for i, _ := range list {
list[i] = pool.knownQueue.fetchOldest()
}
enc, err := rlp.EncodeToBytes(list)
if err == nil {
pool.db.Put(pool.dbKey, enc)
}
}
// removeEntry removes a pool entry when the entry count limit is reached.
// Note that it is called by the new/known queues from which the entry has already
// been removed so removing it from the queues is not necessary.
func (pool *serverPool) removeEntry(entry *poolEntry) {
pool.newSelect.remove((*discoveredEntry)(entry))
pool.knownSelect.remove((*knownEntry)(entry))
entry.removed = true
delete(pool.entries, entry.id)
}
// setRetryDial starts the timer which will enable dialing a certain node again
func (pool *serverPool) setRetryDial(entry *poolEntry) {
delay := longRetryDelay
if entry.shortRetry > 0 {
entry.shortRetry--
delay = shortRetryDelay
}
delay += time.Duration(rand.Int63n(int64(delay) + 1))
entry.delayedRetry = true
go func() {
select {
case <-pool.quit:
case <-time.After(delay):
select {
case <-pool.quit:
case pool.enableRetry <- entry:
}
}
}()
}
// updateCheckDial is called when an entry can potentially be dialed again. It updates
// its selection weights and checks if new dials can/should be made.
func (pool *serverPool) updateCheckDial(entry *poolEntry) {
pool.newSelect.update((*discoveredEntry)(entry))
pool.knownSelect.update((*knownEntry)(entry))
pool.checkDial()
}
// checkDial checks if new dials can/should be made. It tries to select servers both
// based on good statistics and recent discovery.
func (pool *serverPool) checkDial() {
fillWithKnownSelects := !pool.fastDiscover
for pool.knownSelected < targetKnownSelect {
entry := pool.knownSelect.choose()
if entry == nil {
fillWithKnownSelects = false
break
}
pool.dial((*poolEntry)(entry.(*knownEntry)), true)
}
for pool.knownSelected+pool.newSelected < targetServerCount {
entry := pool.newSelect.choose()
if entry == nil {
break
}
pool.dial((*poolEntry)(entry.(*discoveredEntry)), false)
}
if fillWithKnownSelects {
// no more newly discovered nodes to select and since fast discover period
// is over, we probably won't find more in the near future so select more
// known entries if possible
for pool.knownSelected < targetServerCount {
entry := pool.knownSelect.choose()
if entry == nil {
break
}
pool.dial((*poolEntry)(entry.(*knownEntry)), true)
}
}
}
// dial initiates a new connection
func (pool *serverPool) dial(entry *poolEntry, knownSelected bool) {
if entry.state != psNotConnected {
return
}
entry.state = psDialed
entry.knownSelected = knownSelected
if knownSelected {
pool.knownSelected++
} else {
pool.newSelected++
}
addr := entry.addrSelect.choose().(*poolEntryAddress)
glog.V(logger.Debug).Infof("dialing %v out of %v, known: %v", entry.id.String()+"@"+addr.strKey(), len(entry.addr), knownSelected)
entry.dialed = addr
go func() {
pool.server.AddPeer(discover.NewNode(entry.id, addr.ip, addr.port, addr.port))
select {
case <-pool.quit:
case <-time.After(dialTimeout):
select {
case <-pool.quit:
case pool.timeout <- entry:
}
}
}()
}
// checkDialTimeout checks if the node is still in dialed state and if so, resets it
// and adjusts connection statistics accordingly.
func (pool *serverPool) checkDialTimeout(entry *poolEntry) {
if entry.state != psDialed {
return
}
glog.V(logger.Debug).Infof("timeout %v", entry.id.String()+"@"+entry.dialed.strKey())
entry.state = psNotConnected
if entry.knownSelected {
pool.knownSelected--
} else {
pool.newSelected--
}
entry.connectStats.add(0, 1)
entry.dialed.fails++
pool.setRetryDial(entry)
}
const (
psNotConnected = iota
psDialed
psConnected
psRegistered
)
// poolEntry represents a server node and stores its current state and statistics.
type poolEntry struct {
peer *peer
id discover.NodeID
addr map[string]*poolEntryAddress
lastConnected, dialed *poolEntryAddress
addrSelect weightedRandomSelect
lastDiscovered mclock.AbsTime
known, knownSelected bool
connectStats, delayStats poolStats
responseStats, timeoutStats poolStats
state int
regTime mclock.AbsTime
queueIdx int
removed bool
delayedRetry bool
shortRetry int
}
func (e *poolEntry) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{e.id, e.lastConnected.ip, e.lastConnected.port, e.lastConnected.fails, &e.connectStats, &e.delayStats, &e.responseStats, &e.timeoutStats})
}
func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
var entry struct {
ID discover.NodeID
IP net.IP
Port uint16
Fails uint
CStat, DStat, RStat, TStat poolStats
}
if err := s.Decode(&entry); err != nil {
return err
}
addr := &poolEntryAddress{ip: entry.IP, port: entry.Port, fails: entry.Fails, lastSeen: mclock.Now()}
e.id = entry.ID
e.addr = make(map[string]*poolEntryAddress)
e.addr[addr.strKey()] = addr
e.addrSelect = *newWeightedRandomSelect()
e.addrSelect.update(addr)
e.lastConnected = addr
e.connectStats = entry.CStat
e.delayStats = entry.DStat
e.responseStats = entry.RStat
e.timeoutStats = entry.TStat
e.shortRetry = shortRetryCnt
e.known = true
return nil
}
// discoveredEntry implements wrsItem
type discoveredEntry poolEntry
// Weight calculates random selection weight for newly discovered entries
func (e *discoveredEntry) Weight() int64 {
if e.state != psNotConnected || e.delayedRetry {
return 0
}
t := time.Duration(mclock.Now() - e.lastDiscovered)
if t <= discoverExpireStart {
return 1000000000
} else {
return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst)))
}
}
// knownEntry implements wrsItem
type knownEntry poolEntry
// Weight calculates random selection weight for known entries
func (e *knownEntry) Weight() int64 {
if e.state != psNotConnected || !e.known || e.delayedRetry {
return 0
}
return int64(1000000000 * e.connectStats.recentAvg() * math.Exp(-float64(e.lastConnected.fails)*failDropLn-e.responseStats.recentAvg()/float64(responseScoreTC)-e.delayStats.recentAvg()/float64(delayScoreTC)) * math.Pow((1-e.timeoutStats.recentAvg()), timeoutPow))
}
// poolEntryAddress is a separate object because currently it is necessary to remember
// multiple potential network addresses for a pool entry. This will be removed after
// the final implementation of v5 discovery which will retrieve signed and serial
// numbered advertisements, making it clear which IP/port is the latest one.
type poolEntryAddress struct {
ip net.IP
port uint16
lastSeen mclock.AbsTime // last time it was discovered, connected or loaded from db
fails uint // connection failures since last successful connection (persistent)
}
func (a *poolEntryAddress) Weight() int64 {
t := time.Duration(mclock.Now() - a.lastSeen)
return int64(1000000*math.Exp(-float64(t)/float64(discoverExpireConst)-float64(a.fails)*addrFailDropLn)) + 1
}
func (a *poolEntryAddress) strKey() string {
return a.ip.String() + ":" + strconv.Itoa(int(a.port))
}
// poolStats implement statistics for a certain quantity with a long term average
// and a short term value which is adjusted exponentially with a factor of
// pstatRecentAdjust with each update and also returned exponentially to the
// average with the time constant pstatReturnToMeanTC
type poolStats struct {
sum, weight, avg, recent float64
lastRecalc mclock.AbsTime
}
// init initializes stats with a long term sum/update count pair retrieved from the database
func (s *poolStats) init(sum, weight float64) {
s.sum = sum
s.weight = weight
var avg float64
if weight > 0 {
avg = s.sum / weight
}
s.avg = avg
s.recent = avg
s.lastRecalc = mclock.Now()
}
// recalc recalculates recent value return-to-mean and long term average
func (s *poolStats) recalc() {
now := mclock.Now()
s.recent = s.avg + (s.recent-s.avg)*math.Exp(-float64(now-s.lastRecalc)/float64(pstatReturnToMeanTC))
if s.sum == 0 {
s.avg = 0
} else {
if s.sum > s.weight*1e30 {
s.avg = 1e30
} else {
s.avg = s.sum / s.weight
}
}
s.lastRecalc = now
}
// add updates the stats with a new value
func (s *poolStats) add(value, weight float64) {
s.weight += weight
s.sum += value * weight
s.recalc()
}
// recentAvg returns the short-term adjusted average
func (s *poolStats) recentAvg() float64 {
s.recalc()
return s.recent
}
func (s *poolStats) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{math.Float64bits(s.sum), math.Float64bits(s.weight)})
}
func (s *poolStats) DecodeRLP(st *rlp.Stream) error {
var stats struct {
SumUint, WeightUint uint64
}
if err := st.Decode(&stats); err != nil {
return err
}
s.init(math.Float64frombits(stats.SumUint), math.Float64frombits(stats.WeightUint))
return nil
}
// poolEntryQueue keeps track of its least recently accessed entries and removes
// them when the number of entries reaches the limit
type poolEntryQueue struct {
queue map[int]*poolEntry // known nodes indexed by their latest lastConnCnt value
newPtr, oldPtr, maxCnt int
removeFromPool func(*poolEntry)
}
// newPoolEntryQueue returns a new poolEntryQueue
func newPoolEntryQueue(maxCnt int, removeFromPool func(*poolEntry)) poolEntryQueue {
return poolEntryQueue{queue: make(map[int]*poolEntry), maxCnt: maxCnt, removeFromPool: removeFromPool}
}
// fetchOldest returns and removes the least recently accessed entry
func (q *poolEntryQueue) fetchOldest() *poolEntry {
if len(q.queue) == 0 {
return nil
}
for {
if e := q.queue[q.oldPtr]; e != nil {
delete(q.queue, q.oldPtr)
q.oldPtr++
return e
}
q.oldPtr++
}
}
// remove removes an entry from the queue
func (q *poolEntryQueue) remove(entry *poolEntry) {
if q.queue[entry.queueIdx] == entry {
delete(q.queue, entry.queueIdx)
}
}
// setLatest adds or updates a recently accessed entry. It also checks if an old entry
// needs to be removed and removes it from the parent pool too with a callback function.
func (q *poolEntryQueue) setLatest(entry *poolEntry) {
if q.queue[entry.queueIdx] == entry {
delete(q.queue, entry.queueIdx)
} else {
if len(q.queue) == q.maxCnt {
e := q.fetchOldest()
q.remove(e)
q.removeFromPool(e)
}
}
entry.queueIdx = q.newPtr
q.queue[entry.queueIdx] = entry
q.newPtr++
}

11
light/lightchain.go
View file

@ -505,3 +505,14 @@ func (self *LightChain) SyncCht(ctx context.Context) bool {
}
return false
}
// LockChain locks the chain mutex for reading so that multiple canonical hashes can be
// retrieved while it is guaranteed that they belong to the same version of the chain
func (self *LightChain) LockChain() {
self.chainmu.RLock()
}
// UnlockChain unlocks the chain mutex
func (self *LightChain) UnlockChain() {
self.chainmu.RUnlock()
}

15
light/odr.go
View file

@ -48,6 +48,7 @@ type OdrRequest interface {
// TrieID identifies a state or account storage trie
type TrieID struct {
BlockHash, Root common.Hash
BlockNumber uint64
AccKey []byte
}
@ -55,9 +56,10 @@ type TrieID struct {
// header.
func StateTrieID(header *types.Header) *TrieID {
return &TrieID{
BlockHash: header.Hash(),
AccKey: nil,
Root: header.Root,
BlockHash: header.Hash(),
BlockNumber: header.Number.Uint64(),
AccKey: nil,
Root: header.Root,
}
}
@ -66,9 +68,10 @@ func StateTrieID(header *types.Header) *TrieID {
// checking Merkle proofs.
func StorageTrieID(state *TrieID, addr common.Address, root common.Hash) *TrieID {
return &TrieID{
BlockHash: state.BlockHash,
AccKey: crypto.Keccak256(addr[:]),
Root: root,
BlockHash: state.BlockHash,
BlockNumber: state.BlockNumber,
AccKey: crypto.Keccak256(addr[:]),
Root: root,
}
}

5
light/odr_util.go
View file

@ -38,8 +38,9 @@ var (
ErrNoTrustedCht = errors.New("No trusted canonical hash trie")
ErrNoHeader = errors.New("Header not found")
ChtFrequency = uint64(4096)
trustedChtKey = []byte("TrustedCHT")
ChtFrequency = uint64(4096)
ChtConfirmations = uint64(2048)
trustedChtKey = []byte("TrustedCHT")
)
type ChtNode struct {

17
light/txpool.go
View file

@ -346,19 +346,8 @@ func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error
if from, err = types.Sender(pool.signer, tx); err != nil {
return core.ErrInvalidSender
}
// Make sure the account exist. Non existent accounts
// haven't got funds and well therefor never pass.
currentState := pool.currentState()
if h, err := currentState.HasAccount(ctx, from); err == nil {
if !h {
return core.ErrNonExistentAccount
}
} else {
return err
}
// Last but not least check for nonce errors
currentState := pool.currentState()
if n, err := currentState.GetNonce(ctx, from); err == nil {
if n > tx.Nonce() {
return core.ErrNonce
@ -500,7 +489,7 @@ func (tp *TxPool) GetTransaction(hash common.Hash) *types.Transaction {
// GetTransactions returns all currently processable transactions.
// The returned slice may be modified by the caller.
func (self *TxPool) GetTransactions() (txs types.Transactions) {
func (self *TxPool) GetTransactions() (txs types.Transactions, err error) {
self.mu.RLock()
defer self.mu.RUnlock()
@ -510,7 +499,7 @@ func (self *TxPool) GetTransactions() (txs types.Transactions) {
txs[i] = tx
i++
}
return txs
return txs, nil
}
// Content retrieves the data content of the transaction pool, returning all the

10
miner/miner.go
View file

@ -119,15 +119,14 @@ func (m *Miner) SetGasPrice(price *big.Int) {
func (self *Miner) Start(coinbase common.Address, threads int) {
atomic.StoreInt32(&self.shouldStart, 1)
self.threads = threads
self.worker.coinbase = coinbase
self.worker.setEtherbase(coinbase)
self.coinbase = coinbase
self.threads = threads
if atomic.LoadInt32(&self.canStart) == 0 {
glog.V(logger.Info).Infoln("Can not start mining operation due to network sync (starts when finished)")
return
}
atomic.StoreInt32(&self.mining, 1)
for i := 0; i < threads; i++ {
@ -135,9 +134,7 @@ func (self *Miner) Start(coinbase common.Address, threads int) {
}
glog.V(logger.Info).Infof("Starting mining operation (CPU=%d TOT=%d)\n", threads, len(self.worker.agents))
self.worker.start()
self.worker.commitNewWork()
}
@ -177,8 +174,7 @@ func (self *Miner) SetExtra(extra []byte) error {
if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
return fmt.Errorf("Extra exceeds max length. %d > %v", len(extra), params.MaximumExtraDataSize)
}
self.worker.extra = extra
self.worker.setExtra(extra)
return nil
}

58
miner/remote_agent.go
View file

@ -27,6 +27,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/pow"
)
type hashrate struct {
@ -37,10 +38,11 @@ type hashrate struct {
type RemoteAgent struct {
mu sync.Mutex
quit chan struct{}
quitCh chan struct{}
workCh chan *Work
returnCh chan<- *Result
pow pow.PoW
currentWork *Work
work map[common.Hash]*Work
@ -50,8 +52,9 @@ type RemoteAgent struct {
running int32 // running indicates whether the agent is active. Call atomically
}
func NewRemoteAgent() *RemoteAgent {
func NewRemoteAgent(pow pow.PoW) *RemoteAgent {
return &RemoteAgent{
pow: pow,
work: make(map[common.Hash]*Work),
hashrate: make(map[common.Hash]hashrate),
}
@ -76,18 +79,16 @@ func (a *RemoteAgent) Start() {
if !atomic.CompareAndSwapInt32(&a.running, 0, 1) {
return
}
a.quit = make(chan struct{})
a.quitCh = make(chan struct{})
a.workCh = make(chan *Work, 1)
go a.maintainLoop()
go a.loop(a.workCh, a.quitCh)
}
func (a *RemoteAgent) Stop() {
if !atomic.CompareAndSwapInt32(&a.running, 1, 0) {
return
}
close(a.quit)
close(a.quitCh)
close(a.workCh)
}
@ -128,35 +129,46 @@ func (a *RemoteAgent) GetWork() ([3]string, error) {
return res, errors.New("No work available yet, don't panic.")
}
// Returns true or false, but does not indicate if the PoW was correct
// SubmitWork tries to inject a PoW solution tinto the remote agent, returning
// whether the solution was acceted or not (not can be both a bad PoW as well as
// any other error, like no work pending).
func (a *RemoteAgent) SubmitWork(nonce uint64, mixDigest, hash common.Hash) bool {
a.mu.Lock()
defer a.mu.Unlock()
// Make sure the work submitted is present
if a.work[hash] != nil {
block := a.work[hash].Block.WithMiningResult(nonce, mixDigest)
a.returnCh <- &Result{a.work[hash], block}
delete(a.work, hash)
return true
} else {
glog.V(logger.Info).Infof("Work was submitted for %x but no pending work found\n", hash)
work := a.work[hash]
if work == nil {
glog.V(logger.Info).Infof("Work was submitted for %x but no pending work found", hash)
return false
}
// Make sure the PoW solutions is indeed valid
block := work.Block.WithMiningResult(nonce, mixDigest)
if !a.pow.Verify(block) {
glog.V(logger.Warn).Infof("Invalid PoW submitted for %x", hash)
return false
}
// Solutions seems to be valid, return to the miner and notify acceptance
a.returnCh <- &Result{work, block}
delete(a.work, hash)
return false
return true
}
func (a *RemoteAgent) maintainLoop() {
// loop monitors mining events on the work and quit channels, updating the internal
// state of the rmeote miner until a termination is requested.
//
// Note, the reason the work and quit channels are passed as parameters is because
// RemoteAgent.Start() constantly recreates these channels, so the loop code cannot
// assume data stability in these member fields.
func (a *RemoteAgent) loop(workCh chan *Work, quitCh chan struct{}) {
ticker := time.Tick(5 * time.Second)
out:
for {
select {
case <-a.quit:
break out
case work := <-a.workCh:
case <-quitCh:
return
case work := <-workCh:
a.mu.Lock()
a.currentWork = work
a.mu.Unlock()

118
miner/unconfirmed.go Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package miner
import (
"container/ring"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
)
// headerRetriever is used by the unconfirmed block set to verify whether a previously
// mined block is part of the canonical chain or not.
type headerRetriever interface {
// GetHeaderByNumber retrieves the canonical header associated with a block number.
GetHeaderByNumber(number uint64) *types.Header
}
// unconfirmedBlock is a small collection of metadata about a locally mined block
// that is placed into a unconfirmed set for canonical chain inclusion tracking.
type unconfirmedBlock struct {
index uint64
hash common.Hash
}
// unconfirmedBlocks implements a data structure to maintain locally mined blocks
// have have not yet reached enough maturity to guarantee chain inclusion. It is
// used by the miner to provide logs to the user when a previously mined block
// has a high enough guarantee to not be reorged out of te canonical chain.
type unconfirmedBlocks struct {
chain headerRetriever // Blockchain to verify canonical status through
depth uint // Depth after which to discard previous blocks
blocks *ring.Ring // Block infos to allow canonical chain cross checks
lock sync.RWMutex // Protects the fields from concurrent access
}
// newUnconfirmedBlocks returns new data structure to track currently unconfirmed blocks.
func newUnconfirmedBlocks(chain headerRetriever, depth uint) *unconfirmedBlocks {
return &unconfirmedBlocks{
chain: chain,
depth: depth,
}
}
// Insert adds a new block to the set of unconfirmed ones.
func (set *unconfirmedBlocks) Insert(index uint64, hash common.Hash) {
// If a new block was mined locally, shift out any old enough blocks
set.Shift(index)
// Create the new item as its own ring
item := ring.New(1)
item.Value = &unconfirmedBlock{
index: index,
hash: hash,
}
// Set as the initial ring or append to the end
set.lock.Lock()
defer set.lock.Unlock()
if set.blocks == nil {
set.blocks = item
} else {
set.blocks.Move(-1).Link(item)
}
// Display a log for the user to notify of a new mined block unconfirmed
glog.V(logger.Info).Infof("🔨 mined potential block #%d [%x…], waiting for %d blocks to confirm", index, hash.Bytes()[:4], set.depth)
}
// Shift drops all unconfirmed blocks from the set which exceed the unconfirmed sets depth
// allowance, checking them against the canonical chain for inclusion or staleness
// report.
func (set *unconfirmedBlocks) Shift(height uint64) {
set.lock.Lock()
defer set.lock.Unlock()
for set.blocks != nil {
// Retrieve the next unconfirmed block and abort if too fresh
next := set.blocks.Value.(*unconfirmedBlock)
if next.index+uint64(set.depth) > height {
break
}
// Block seems to exceed depth allowance, check for canonical status
header := set.chain.GetHeaderByNumber(next.index)
switch {
case header == nil:
glog.V(logger.Warn).Infof("failed to retrieve header of mined block #%d [%x…]", next.index, next.hash.Bytes()[:4])
case header.Hash() == next.hash:
glog.V(logger.Info).Infof("🔗 mined block #%d [%x…] reached canonical chain", next.index, next.hash.Bytes()[:4])
default:
glog.V(logger.Info).Infof("⑂ mined block #%d [%x…] became a side fork", next.index, next.hash.Bytes()[:4])
}
// Drop the block out of the ring
if set.blocks.Value == set.blocks.Next().Value {
set.blocks = nil
} else {
set.blocks = set.blocks.Move(-1)
set.blocks.Unlink(1)
set.blocks = set.blocks.Move(1)
}
}
}

85
miner/unconfirmed_test.go Normal file
View file

@ -0,0 +1,85 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package miner
import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
// noopHeaderRetriever is an implementation of headerRetriever that always
// returns nil for any requested headers.
type noopHeaderRetriever struct{}
func (r *noopHeaderRetriever) GetHeaderByNumber(number uint64) *types.Header {
return nil
}
// Tests that inserting blocks into the unconfirmed set accumulates them until
// the desired depth is reached, after which they begin to be dropped.
func TestUnconfirmedInsertBounds(t *testing.T) {
limit := uint(10)
pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit)
for depth := uint64(0); depth < 2*uint64(limit); depth++ {
// Insert multiple blocks for the same level just to stress it
for i := 0; i < int(depth); i++ {
pool.Insert(depth, common.Hash([32]byte{byte(depth), byte(i)}))
}
// Validate that no blocks below the depth allowance are left in
pool.blocks.Do(func(block interface{}) {
if block := block.(*unconfirmedBlock); block.index+uint64(limit) <= depth {
t.Errorf("depth %d: block %x not dropped", depth, block.hash)
}
})
}
}
// Tests that shifting blocks out of the unconfirmed set works both for normal
// cases as well as for corner cases such as empty sets, empty shifts or full
// shifts.
func TestUnconfirmedShifts(t *testing.T) {
// Create a pool with a few blocks on various depths
limit, start := uint(10), uint64(25)
pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit)
for depth := start; depth < start+uint64(limit); depth++ {
pool.Insert(depth, common.Hash([32]byte{byte(depth)}))
}
// Try to shift below the limit and ensure no blocks are dropped
pool.Shift(start + uint64(limit) - 1)
if n := pool.blocks.Len(); n != int(limit) {
t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit)
}
// Try to shift half the blocks out and verify remainder
pool.Shift(start + uint64(limit) - 1 + uint64(limit/2))
if n := pool.blocks.Len(); n != int(limit)/2 {
t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit/2)
}
// Try to shift all the remaining blocks out and verify emptyness
pool.Shift(start + 2*uint64(limit))
if n := pool.blocks.Len(); n != 0 {
t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0)
}
// Try to shift out from the empty set and make sure it doesn't break
pool.Shift(start + 3*uint64(limit))
if n := pool.blocks.Len(); n != 0 {
t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0)
}
}

103
miner/worker.go
View file

@ -55,26 +55,20 @@ type Agent interface {
GetHashRate() int64
}
type uint64RingBuffer struct {
ints []uint64 //array of all integers in buffer
next int //where is the next insertion? assert 0 <= next < len(ints)
}
// Work is the workers current environment and holds
// all of the current state information
type Work struct {
config *params.ChainConfig
signer types.Signer
state *state.StateDB // apply state changes here
ancestors *set.Set // ancestor set (used for checking uncle parent validity)
family *set.Set // family set (used for checking uncle invalidity)
uncles *set.Set // uncle set
tcount int // tx count in cycle
ownedAccounts *set.Set
lowGasTxs types.Transactions
failedTxs types.Transactions
localMinedBlocks *uint64RingBuffer // the most recent block numbers that were mined locally (used to check block inclusion)
state *state.StateDB // apply state changes here
ancestors *set.Set // ancestor set (used for checking uncle parent validity)
family *set.Set // family set (used for checking uncle invalidity)
uncles *set.Set // uncle set
tcount int // tx count in cycle
ownedAccounts *set.Set
lowGasTxs types.Transactions
failedTxs types.Transactions
Block *types.Block // the new block
@ -123,6 +117,8 @@ type worker struct {
txQueueMu sync.Mutex
txQueue map[common.Hash]*types.Transaction
unconfirmed *unconfirmedBlocks // set of locally mined blocks pending canonicalness confirmations
// atomic status counters
mining int32
atWork int32
@ -144,6 +140,7 @@ func newWorker(config *params.ChainConfig, coinbase common.Address, eth Backend,
coinbase: coinbase,
txQueue: make(map[common.Hash]*types.Transaction),
agents: make(map[Agent]struct{}),
unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), 5),
fullValidation: false,
}
worker.events = worker.mux.Subscribe(core.ChainHeadEvent{}, core.ChainSideEvent{}, core.TxPreEvent{})
@ -161,6 +158,12 @@ func (self *worker) setEtherbase(addr common.Address) {
self.coinbase = addr
}
func (self *worker) setExtra(extra []byte) {
self.mu.Lock()
defer self.mu.Unlock()
self.extra = extra
}
func (self *worker) pending() (*types.Block, *state.StateDB) {
self.currentMu.Lock()
defer self.currentMu.Unlock()
@ -263,18 +266,6 @@ func (self *worker) update() {
}
}
func newLocalMinedBlock(blockNumber uint64, prevMinedBlocks *uint64RingBuffer) (minedBlocks *uint64RingBuffer) {
if prevMinedBlocks == nil {
minedBlocks = &uint64RingBuffer{next: 0, ints: make([]uint64, miningLogAtDepth+1)}
} else {
minedBlocks = prevMinedBlocks
}
minedBlocks.ints[minedBlocks.next] = blockNumber
minedBlocks.next = (minedBlocks.next + 1) % len(minedBlocks.ints)
return minedBlocks
}
func (self *worker) wait() {
for {
mustCommitNewWork := true
@ -349,17 +340,8 @@ func (self *worker) wait() {
}
}(block, work.state.Logs(), work.receipts)
}
// check staleness and display confirmation
var stale, confirm string
canonBlock := self.chain.GetBlockByNumber(block.NumberU64())
if canonBlock != nil && canonBlock.Hash() != block.Hash() {
stale = "stale "
} else {
confirm = "Wait 5 blocks for confirmation"
work.localMinedBlocks = newLocalMinedBlock(block.Number().Uint64(), work.localMinedBlocks)
}
glog.V(logger.Info).Infof("🔨 Mined %sblock (#%v / %x). %s", stale, block.Number(), block.Hash().Bytes()[:4], confirm)
// Insert the block into the set of pending ones to wait for confirmations
self.unconfirmed.Insert(block.NumberU64(), block.Hash())
if mustCommitNewWork {
self.commitNewWork()
@ -411,9 +393,6 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error
// Keep track of transactions which return errors so they can be removed
work.tcount = 0
work.ownedAccounts = accountAddressesSet(accounts)
if self.current != nil {
work.localMinedBlocks = self.current.localMinedBlocks
}
self.current = work
return nil
}
@ -429,38 +408,6 @@ func (w *worker) setGasPrice(p *big.Int) {
w.mux.Post(core.GasPriceChanged{Price: w.gasPrice})
}
func (self *worker) isBlockLocallyMined(current *Work, deepBlockNum uint64) bool {
//Did this instance mine a block at {deepBlockNum} ?
var isLocal = false
for idx, blockNum := range current.localMinedBlocks.ints {
if deepBlockNum == blockNum {
isLocal = true
current.localMinedBlocks.ints[idx] = 0 //prevent showing duplicate logs
break
}
}
//Short-circuit on false, because the previous and following tests must both be true
if !isLocal {
return false
}
//Does the block at {deepBlockNum} send earnings to my coinbase?
var block = self.chain.GetBlockByNumber(deepBlockNum)
return block != nil && block.Coinbase() == self.coinbase
}
func (self *worker) logLocalMinedBlocks(current, previous *Work) {
if previous != nil && current.localMinedBlocks != nil {
nextBlockNum := current.Block.NumberU64()
for checkBlockNum := previous.Block.NumberU64(); checkBlockNum < nextBlockNum; checkBlockNum++ {
inspectBlockNum := checkBlockNum - miningLogAtDepth
if self.isBlockLocallyMined(current, inspectBlockNum) {
glog.V(logger.Info).Infof("🔨 🔗 Mined %d blocks back: block #%v", miningLogAtDepth, inspectBlockNum)
}
}
}
}
func (self *worker) commitNewWork() {
self.mu.Lock()
defer self.mu.Unlock()
@ -507,7 +454,6 @@ func (self *worker) commitNewWork() {
}
}
}
previous := self.current
// Could potentially happen if starting to mine in an odd state.
err := self.makeCurrent(parent, header)
if err != nil {
@ -519,7 +465,14 @@ func (self *worker) commitNewWork() {
if self.config.DAOForkSupport && self.config.DAOForkBlock != nil && self.config.DAOForkBlock.Cmp(header.Number) == 0 {
core.ApplyDAOHardFork(work.state)
}
txs := types.NewTransactionsByPriceAndNonce(self.eth.TxPool().Pending())
pending, err := self.eth.TxPool().Pending()
if err != nil {
glog.Errorf("Could not fetch pending transactions: %v", err)
return
}
txs := types.NewTransactionsByPriceAndNonce(pending)
work.commitTransactions(self.mux, txs, self.gasPrice, self.chain)
self.eth.TxPool().RemoveBatch(work.lowGasTxs)
@ -561,7 +514,7 @@ func (self *worker) commitNewWork() {
// We only care about logging if we're actually mining.
if atomic.LoadInt32(&self.mining) == 1 {
glog.V(logger.Info).Infof("commit new work on block %v with %d txs & %d uncles. Took %v\n", work.Block.Number(), work.tcount, len(uncles), time.Since(tstart))
self.logLocalMinedBlocks(work, previous)
self.unconfirmed.Shift(work.Block.NumberU64() - 1)
}
self.push(work)
}

2
mobile/params.go
View file

@ -50,7 +50,7 @@ func TestnetChainConfig() *ChainConfig {
ChainID: params.TestNetChainID.Int64(),
HomesteadBlock: params.TestNetHomesteadBlock.Int64(),
DAOForkBlock: 0,
DAOForkSupport: true,
DAOForkSupport: false,
EIP150Block: params.TestNetHomesteadGasRepriceBlock.Int64(),
EIP150Hash: Hash{params.TestNetHomesteadGasRepriceHash},
EIP155Block: params.TestNetSpuriousDragon.Int64(),

2
p2p/discover/udp_test.go
View file

@ -374,7 +374,7 @@ func TestUDP_successfulPing(t *testing.T) {
if n.ID != rid {
t.Errorf("node has wrong ID: got %v, want %v", n.ID, rid)
}
if !bytes.Equal(n.IP, test.remoteaddr.IP) {
if !n.IP.Equal(test.remoteaddr.IP) {
t.Errorf("node has wrong IP: got %v, want: %v", n.IP, test.remoteaddr.IP)
}
if int(n.UDP) != test.remoteaddr.Port {

121
p2p/discv5/net.go
View file

@ -126,8 +126,15 @@ type topicRegisterReq struct {
}
type topicSearchReq struct {
topic Topic
found chan<- string
topic Topic
found chan<- *Node
lookup chan<- bool
delay time.Duration
}
type topicSearchResult struct {
target lookupInfo
nodes []*Node
}
type timeoutEvent struct {
@ -263,16 +270,23 @@ func (net *Network) lookup(target common.Hash, stopOnMatch bool) []*Node {
break
}
// Wait for the next reply.
for _, n := range <-reply {
if n != nil && !seen[n.ID] {
seen[n.ID] = true
result.push(n, bucketSize)
if stopOnMatch && n.sha == target {
return result.entries
select {
case nodes := <-reply:
for _, n := range nodes {
if n != nil && !seen[n.ID] {
seen[n.ID] = true
result.push(n, bucketSize)
if stopOnMatch && n.sha == target {
return result.entries
}
}
}
pendingQueries--
case <-time.After(respTimeout):
// forget all pending requests, start new ones
pendingQueries = 0
reply = make(chan []*Node, alpha)
}
pendingQueries--
}
return result.entries
}
@ -293,18 +307,20 @@ func (net *Network) RegisterTopic(topic Topic, stop <-chan struct{}) {
}
}
func (net *Network) SearchTopic(topic Topic, stop <-chan struct{}, found chan<- string) {
select {
case net.topicSearchReq <- topicSearchReq{topic, found}:
case <-net.closed:
return
}
select {
case <-net.closed:
case <-stop:
func (net *Network) SearchTopic(topic Topic, setPeriod <-chan time.Duration, found chan<- *Node, lookup chan<- bool) {
for {
select {
case net.topicSearchReq <- topicSearchReq{topic, nil}:
case <-net.closed:
return
case delay, ok := <-setPeriod:
select {
case net.topicSearchReq <- topicSearchReq{topic: topic, found: found, lookup: lookup, delay: delay}:
case <-net.closed:
return
}
if !ok {
return
}
}
}
}
@ -347,6 +363,13 @@ func (net *Network) reqTableOp(f func()) (called bool) {
// TODO: external address handling.
type topicSearchInfo struct {
lookupChn chan<- bool
period time.Duration
}
const maxSearchCount = 5
func (net *Network) loop() {
var (
refreshTimer = time.NewTicker(autoRefreshInterval)
@ -385,10 +408,12 @@ func (net *Network) loop() {
topicRegisterLookupTarget lookupInfo
topicRegisterLookupDone chan []*Node
topicRegisterLookupTick = time.NewTimer(0)
topicSearchLookupTarget lookupInfo
searchReqWhenRefreshDone []topicSearchReq
searchInfo = make(map[Topic]topicSearchInfo)
activeSearchCount int
)
topicSearchLookupDone := make(chan []*Node, 1)
topicSearchLookupDone := make(chan topicSearchResult, 100)
topicSearch := make(chan Topic, 100)
<-topicRegisterLookupTick.C
statsDump := time.NewTicker(10 * time.Second)
@ -504,21 +529,52 @@ loop:
case req := <-net.topicSearchReq:
if refreshDone == nil {
debugLog("<-net.topicSearchReq")
if req.found == nil {
net.ticketStore.removeSearchTopic(req.topic)
info, ok := searchInfo[req.topic]
if ok {
if req.delay == time.Duration(0) {
delete(searchInfo, req.topic)
net.ticketStore.removeSearchTopic(req.topic)
} else {
info.period = req.delay
searchInfo[req.topic] = info
}
continue
}
net.ticketStore.addSearchTopic(req.topic, req.found)
if (topicSearchLookupTarget.target == common.Hash{}) {
topicSearchLookupDone <- nil
if req.delay != time.Duration(0) {
var info topicSearchInfo
info.period = req.delay
info.lookupChn = req.lookup
searchInfo[req.topic] = info
net.ticketStore.addSearchTopic(req.topic, req.found)
topicSearch <- req.topic
}
} else {
searchReqWhenRefreshDone = append(searchReqWhenRefreshDone, req)
}
case nodes := <-topicSearchLookupDone:
debugLog("<-topicSearchLookupDone")
net.ticketStore.searchLookupDone(topicSearchLookupTarget, nodes, func(n *Node) []byte {
case topic := <-topicSearch:
if activeSearchCount < maxSearchCount {
activeSearchCount++
target := net.ticketStore.nextSearchLookup(topic)
go func() {
nodes := net.lookup(target.target, false)
topicSearchLookupDone <- topicSearchResult{target: target, nodes: nodes}
}()
}
period := searchInfo[topic].period
if period != time.Duration(0) {
go func() {
time.Sleep(period)
topicSearch <- topic
}()
}
case res := <-topicSearchLookupDone:
activeSearchCount--
if lookupChn := searchInfo[res.target.topic].lookupChn; lookupChn != nil {
lookupChn <- net.ticketStore.radius[res.target.topic].converged
}
net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node) []byte {
net.ping(n, n.addr())
return n.pingEcho
}, func(n *Node, topic Topic) []byte {
@ -531,11 +587,6 @@ loop:
return nil
}
})
topicSearchLookupTarget = net.ticketStore.nextSearchLookup()
target := topicSearchLookupTarget.target
if (target != common.Hash{}) {
go func() { topicSearchLookupDone <- net.lookup(target, false) }()
}
case <-statsDump.C:
debugLog("<-statsDump.C")
@ -708,7 +759,7 @@ func (net *Network) internNodeFromNeighbours(sender *net.UDPAddr, rn rpcNode) (n
}
return n, err
}
if !bytes.Equal(n.IP, rn.IP) || n.UDP != rn.UDP || n.TCP != rn.TCP {
if !n.IP.Equal(rn.IP) || n.UDP != rn.UDP || n.TCP != rn.TCP {
err = fmt.Errorf("metadata mismatch: got %v, want %v", rn, n)
}
return n, err

3
p2p/discv5/node.go
View file

@ -17,7 +17,6 @@
package discv5
import (
"bytes"
"crypto/ecdsa"
"crypto/elliptic"
"encoding/hex"
@ -81,7 +80,7 @@ func (n *Node) addrEqual(a *net.UDPAddr) bool {
if ipv4 := a.IP.To4(); ipv4 != nil {
ip = ipv4
}
return n.UDP == uint16(a.Port) && bytes.Equal(n.IP, ip)
return n.UDP == uint16(a.Port) && n.IP.Equal(ip)
}
// Incomplete returns true for nodes with no IP address.

44
p2p/discv5/ticket.go
View file

@ -138,16 +138,12 @@ type ticketStore struct {
nextTicketReg mclock.AbsTime
searchTopicMap map[Topic]searchTopic
searchTopicList []Topic
searchTopicPtr int
nextTopicQueryCleanup mclock.AbsTime
queriesSent map[*Node]map[common.Hash]sentQuery
radiusLookupCnt int
}
type searchTopic struct {
foundChn chan<- string
listIdx int
foundChn chan<- *Node
}
type sentQuery struct {
@ -183,23 +179,15 @@ func (s *ticketStore) addTopic(t Topic, register bool) {
}
}
func (s *ticketStore) addSearchTopic(t Topic, foundChn chan<- string) {
func (s *ticketStore) addSearchTopic(t Topic, foundChn chan<- *Node) {
s.addTopic(t, false)
if s.searchTopicMap[t].foundChn == nil {
s.searchTopicList = append(s.searchTopicList, t)
s.searchTopicMap[t] = searchTopic{foundChn: foundChn, listIdx: len(s.searchTopicList) - 1}
s.searchTopicMap[t] = searchTopic{foundChn: foundChn}
}
}
func (s *ticketStore) removeSearchTopic(t Topic) {
if st := s.searchTopicMap[t]; st.foundChn != nil {
lastIdx := len(s.searchTopicList) - 1
lastTopic := s.searchTopicList[lastIdx]
s.searchTopicList[st.listIdx] = lastTopic
sl := s.searchTopicMap[lastTopic]
sl.listIdx = st.listIdx
s.searchTopicMap[lastTopic] = sl
s.searchTopicList = s.searchTopicList[:lastIdx]
delete(s.searchTopicMap, t)
}
}
@ -247,20 +235,13 @@ func (s *ticketStore) nextRegisterLookup() (lookup lookupInfo, delay time.Durati
return lookupInfo{}, 40 * time.Second
}
func (s *ticketStore) nextSearchLookup() lookupInfo {
if len(s.searchTopicList) == 0 {
return lookupInfo{}
}
if s.searchTopicPtr >= len(s.searchTopicList) {
s.searchTopicPtr = 0
}
topic := s.searchTopicList[s.searchTopicPtr]
s.searchTopicPtr++
target := s.radius[topic].nextTarget(s.radiusLookupCnt >= searchForceQuery)
func (s *ticketStore) nextSearchLookup(topic Topic) lookupInfo {
tr := s.radius[topic]
target := tr.nextTarget(tr.radiusLookupCnt >= searchForceQuery)
if target.radiusLookup {
s.radiusLookupCnt++
tr.radiusLookupCnt++
} else {
s.radiusLookupCnt = 0
tr.radiusLookupCnt = 0
}
return target
}
@ -662,9 +643,9 @@ func (s *ticketStore) gotTopicNodes(from *Node, hash common.Hash, nodes []rpcNod
if ip.IsUnspecified() || ip.IsLoopback() {
ip = from.IP
}
enode := NewNode(node.ID, ip, node.UDP-1, node.TCP-1).String() // subtract one from port while discv5 is running in test mode on UDPport+1
n := NewNode(node.ID, ip, node.UDP-1, node.TCP-1) // subtract one from port while discv5 is running in test mode on UDPport+1
select {
case chn <- enode:
case chn <- n:
default:
return false
}
@ -677,6 +658,8 @@ type topicRadius struct {
topicHashPrefix uint64
radius, minRadius uint64
buckets []topicRadiusBucket
converged bool
radiusLookupCnt int
}
type topicRadiusEvent int
@ -706,7 +689,7 @@ func (b *topicRadiusBucket) update(now mclock.AbsTime) {
b.lastTime = now
for target, tm := range b.lookupSent {
if now-tm > mclock.AbsTime(pingTimeout) {
if now-tm > mclock.AbsTime(respTimeout) {
b.weights[trNoAdjust] += 1
delete(b.lookupSent, target)
}
@ -906,6 +889,7 @@ func (r *topicRadius) recalcRadius() (radius uint64, radiusLookup int) {
if radiusLookup == -1 {
// no more radius lookups needed at the moment, return a radius
r.converged = true
rad := maxBucket
if minRadBucket < rad {
rad = minRadBucket

2
p2p/discv5/udp.go
View file

@ -196,7 +196,7 @@ func makeEndpoint(addr *net.UDPAddr, tcpPort uint16) rpcEndpoint {
}
func (e1 rpcEndpoint) equal(e2 rpcEndpoint) bool {
return e1.UDP == e2.UDP && e1.TCP == e2.TCP && bytes.Equal(e1.IP, e2.IP)
return e1.UDP == e2.UDP && e1.TCP == e2.TCP && e1.IP.Equal(e2.IP)
}
func nodeFromRPC(sender *net.UDPAddr, rn rpcNode) (*Node, error) {

3
p2p/nat/nat_test.go
View file

@ -17,7 +17,6 @@
package nat
import (
"bytes"
"net"
"testing"
"time"
@ -56,7 +55,7 @@ func TestAutoDiscRace(t *testing.T) {
t.Errorf("result %d: unexpected error: %v", i, rval.err)
}
wantIP := net.IP{33, 44, 55, 66}
if !bytes.Equal(rval.ip, wantIP) {
if !rval.ip.Equal(wantIP) {
t.Errorf("result %d: got IP %v, want %v", i, rval.ip, wantIP)
}
}

2
params/version.go
View file

@ -21,7 +21,7 @@ import "fmt"
const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 5 // Minor version component of the current release
VersionPatch = 5 // Patch version component of the current release
VersionPatch = 6 // Patch version component of the current release
VersionMeta = "unstable" // Version metadata to append to the version string
)

2
swarm/network/protocol.go
View file

@ -51,7 +51,7 @@ const (
Version = 0
ProtocolLength = uint64(8)
ProtocolMaxMsgSize = 10 * 1024 * 1024
NetworkId = 322
NetworkId = 3
)
const (

16
whisper/shhapi/api.go
View file

@ -55,6 +55,22 @@ func APIs() []rpc.API {
}
}
// Start starts the Whisper worker threads.
func (api *PublicWhisperAPI) Start() error {
if api.whisper == nil {
return whisperOffLineErr
}
return api.whisper.Start(nil)
}
// Stop stops the Whisper worker threads.
func (api *PublicWhisperAPI) Stop() error {
if api.whisper == nil {
return whisperOffLineErr
}
return api.whisper.Stop()
}
// Version returns the Whisper version this node offers.
func (api *PublicWhisperAPI) Version() (*rpc.HexNumber, error) {
if api.whisper == nil {

9
whisper/shhapi/api_test.go
View file

@ -277,6 +277,9 @@ func TestIntegrationAsym(t *testing.T) {
t.Fatalf("failed to create API.")
}
api.Start()
defer api.Stop()
sig, err := api.NewIdentity()
if err != nil {
t.Fatalf("failed NewIdentity: %s.", err)
@ -375,6 +378,9 @@ func TestIntegrationSym(t *testing.T) {
t.Fatalf("failed to create API.")
}
api.Start()
defer api.Stop()
keyname := "schluessel"
err := api.GenerateSymKey(keyname)
if err != nil {
@ -471,6 +477,9 @@ func TestIntegrationSymWithFilter(t *testing.T) {
t.Fatalf("failed to create API.")
}
api.Start()
defer api.Stop()
keyname := "schluessel"
err := api.GenerateSymKey(keyname)
if err != nil {

19
whisper/whisperv5/doc.go
View file

@ -15,9 +15,7 @@
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
Package whisper implements the Whisper PoC-1.
(https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec)
Package whisper implements the Whisper protocol (version 5).
Whisper combines aspects of both DHTs and datagram messaging systems (e.g. UDP).
As such it may be likened and compared to both, not dissimilar to the
@ -42,11 +40,11 @@ const (
ProtocolVersionStr = "5.0"
ProtocolName = "shh"
statusCode = 0
messagesCode = 1
p2pCode = 2
mailRequestCode = 3
NumberOfMessageCodes = 32
statusCode = 0 // used by whisper protocol
messagesCode = 1 // normal whisper message
p2pCode = 2 // peer-to-peer message (to be consumed by the peer, but not forwarded any futher)
p2pRequestCode = 3 // peer-to-peer message, used by Dapp protocol
NumberOfMessageCodes = 64
paddingMask = byte(3)
signatureFlag = byte(4)
@ -57,11 +55,12 @@ const (
saltLength = 12
AESNonceMaxLength = 12
MaxMessageLength = 0xFFFF // todo: remove this restriction after testing in morden and analizing stats. this should be regulated by MinimumPoW.
MinimumPoW = 10.0 // todo: review
MaxMessageLength = 0xFFFF // todo: remove this restriction after testing. this should be regulated by PoW.
MinimumPoW = 1.0 // todo: review after testing.
padSizeLimitLower = 128 // it can not be less - we don't want to reveal the absence of signature
padSizeLimitUpper = 256 // just an arbitrary number, could be changed without losing compatibility
messageQueueLimit = 1024
expirationCycle = time.Second
transmissionCycle = 300 * time.Millisecond

18
whisper/whisperv5/envelope.go
View file

@ -14,14 +14,14 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains the Whisper protocol Envelope element. For formal details please see
// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#envelopes.
// Contains the Whisper protocol Envelope element.
package whisperv5
import (
"crypto/ecdsa"
"encoding/binary"
"errors"
"fmt"
"math"
"time"
@ -86,8 +86,8 @@ func (e *Envelope) Ver() uint64 {
// Seal closes the envelope by spending the requested amount of time as a proof
// of work on hashing the data.
func (e *Envelope) Seal(options *MessageParams) {
var target int
func (e *Envelope) Seal(options *MessageParams) error {
var target, bestBit int
if options.PoW == 0 {
// adjust for the duration of Seal() execution only if execution time is predefined unconditionally
e.Expiry += options.WorkTime
@ -99,7 +99,7 @@ func (e *Envelope) Seal(options *MessageParams) {
h := crypto.Keccak256(e.rlpWithoutNonce())
copy(buf[:32], h)
finish, bestBit := time.Now().Add(time.Duration(options.WorkTime)*time.Second).UnixNano(), 0
finish := time.Now().Add(time.Duration(options.WorkTime) * time.Second).UnixNano()
for nonce := uint64(0); time.Now().UnixNano() < finish; {
for i := 0; i < 1024; i++ {
binary.BigEndian.PutUint64(buf[56:], nonce)
@ -108,12 +108,18 @@ func (e *Envelope) Seal(options *MessageParams) {
if firstBit > bestBit {
e.EnvNonce, bestBit = nonce, firstBit
if target > 0 && bestBit >= target {
return
return nil
}
}
nonce++
}
}
if target > 0 && bestBit < target {
return errors.New("Failed to reach the PoW target")
}
return nil
}
func (e *Envelope) PoW() float64 {

20
whisper/whisperv5/filter.go
View file

@ -21,6 +21,8 @@ import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
)
type Filter struct {
@ -75,11 +77,12 @@ func (fs *Filters) Get(i uint32) *Filter {
return fs.watchers[i]
}
func (fs *Filters) NotifyWatchers(env *Envelope, messageCode uint64) {
func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) {
fs.mutex.RLock()
var msg *ReceivedMessage
for _, watcher := range fs.watchers {
if messageCode == p2pCode && !watcher.AcceptP2P {
for j, watcher := range fs.watchers {
if p2pMessage && !watcher.AcceptP2P {
glog.V(logger.Detail).Infof("msg [%x], filter [%d]: p2p messages are not allowed \n", env.Hash(), j)
continue
}
@ -90,6 +93,11 @@ func (fs *Filters) NotifyWatchers(env *Envelope, messageCode uint64) {
match = watcher.MatchEnvelope(env)
if match {
msg = env.Open(watcher)
if msg == nil {
glog.V(logger.Detail).Infof("msg [%x], filter [%d]: failed to open \n", env.Hash(), j)
}
} else {
glog.V(logger.Detail).Infof("msg [%x], filter [%d]: does not match \n", env.Hash(), j)
}
}
@ -137,12 +145,12 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
if f.PoW > 0 && msg.PoW < f.PoW {
return false
}
if f.Src != nil && !isPubKeyEqual(msg.Src, f.Src) {
if f.Src != nil && !IsPubKeyEqual(msg.Src, f.Src) {
return false
}
if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() {
return isPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic)
return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic)
} else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() {
return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic)
}
@ -176,7 +184,7 @@ func (f *Filter) MatchTopic(topic TopicType) bool {
return false
}
func isPubKeyEqual(a, b *ecdsa.PublicKey) bool {
func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool {
if !ValidatePublicKey(a) {
return false
} else if !ValidatePublicKey(b) {

12
whisper/whisperv5/filter_test.go
View file

@ -139,7 +139,7 @@ func TestComparePubKey(t *testing.T) {
if err != nil {
t.Fatalf("failed to generate second key with seed %d: %s.", seed, err)
}
if isPubKeyEqual(&key1.PublicKey, &key2.PublicKey) {
if IsPubKeyEqual(&key1.PublicKey, &key2.PublicKey) {
t.Fatalf("public keys are equal, seed %d.", seed)
}
@ -149,7 +149,7 @@ func TestComparePubKey(t *testing.T) {
if err != nil {
t.Fatalf("failed to generate third key with seed %d: %s.", seed, err)
}
if isPubKeyEqual(&key1.PublicKey, &key3.PublicKey) {
if IsPubKeyEqual(&key1.PublicKey, &key3.PublicKey) {
t.Fatalf("key1 == key3, seed %d.", seed)
}
}
@ -540,7 +540,7 @@ func TestWatchers(t *testing.T) {
}
for i = 0; i < NumMessages; i++ {
filters.NotifyWatchers(envelopes[i], messagesCode)
filters.NotifyWatchers(envelopes[i], false)
}
var total int
@ -593,7 +593,7 @@ func TestWatchers(t *testing.T) {
}
for i = 0; i < NumMessages; i++ {
filters.NotifyWatchers(envelopes[i], messagesCode)
filters.NotifyWatchers(envelopes[i], false)
}
for i = 0; i < NumFilters; i++ {
@ -629,7 +629,7 @@ func TestWatchers(t *testing.T) {
// test AcceptP2P
total = 0
filters.NotifyWatchers(envelopes[0], p2pCode)
filters.NotifyWatchers(envelopes[0], true)
for i = 0; i < NumFilters; i++ {
mail = tst[i].f.Retrieve()
@ -646,7 +646,7 @@ func TestWatchers(t *testing.T) {
}
f.AcceptP2P = true
total = 0
filters.NotifyWatchers(envelopes[0], p2pCode)
filters.NotifyWatchers(envelopes[0], true)
for i = 0; i < NumFilters; i++ {
mail = tst[i].f.Retrieve()

10
whisper/whisperv5/message.go
View file

@ -14,9 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains the Whisper protocol Message element. For formal details please see
// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#messages.
// todo: fix the spec link, and move it to doc.go
// Contains the Whisper protocol Message element.
package whisperv5
@ -256,7 +254,11 @@ func (msg *SentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er
}
envelope = NewEnvelope(options.TTL, options.Topic, salt, nonce, msg)
envelope.Seal(options)
err = envelope.Seal(options)
if err != nil {
return nil, err
}
return envelope, nil
}

51
whisper/whisperv5/message_test.go
View file

@ -30,11 +30,15 @@ func copyFromBuf(dst []byte, src []byte, beg int) int {
}
func generateMessageParams() (*MessageParams, error) {
// set all the parameters except p.Dst
buf := make([]byte, 1024)
randomize(buf)
sz := rand.Intn(400)
var p MessageParams
p.PoW = 0.01
p.WorkTime = 1
p.TTL = uint32(rand.Intn(1024))
p.Payload = make([]byte, sz)
p.Padding = make([]byte, padSizeLimitUpper)
@ -52,8 +56,6 @@ func generateMessageParams() (*MessageParams, error) {
return nil, err
}
// p.Dst, p.PoW, p.WorkTime are not set
p.PoW = 0.01
return &p, nil
}
@ -114,7 +116,7 @@ func singleMessageTest(t *testing.T, symmetric bool) {
if len(decrypted.Signature) != signatureLength {
t.Fatalf("failed with seed %d: signature len %d.", seed, len(decrypted.Signature))
}
if !isPubKeyEqual(decrypted.Src, &params.Src.PublicKey) {
if !IsPubKeyEqual(decrypted.Src, &params.Src.PublicKey) {
t.Fatalf("failed with seed %d: signature mismatch.", seed)
}
}
@ -152,6 +154,16 @@ func TestMessageWrap(t *testing.T) {
if pow < target {
t.Fatalf("failed Wrap with seed %d: pow < target (%f vs. %f).", seed, pow, target)
}
// set PoW target too high, expect error
msg2 := NewSentMessage(params)
params.TTL = 1000000
params.WorkTime = 1
params.PoW = 10000000.0
env, err = msg2.Wrap(params)
if err == nil {
t.Fatalf("unexpectedly reached the PoW target with seed %d.", seed)
}
}
func TestMessageSeal(t *testing.T) {
@ -256,7 +268,7 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) {
if len(decrypted.Signature) != signatureLength {
t.Fatalf("failed with seed %d: signature len %d.", seed, len(decrypted.Signature))
}
if !isPubKeyEqual(decrypted.Src, &params.Src.PublicKey) {
if !IsPubKeyEqual(decrypted.Src, &params.Src.PublicKey) {
t.Fatalf("failed with seed %d: signature mismatch.", seed)
}
if decrypted.isAsymmetricEncryption() == symmetric {
@ -269,8 +281,37 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) {
if decrypted.Dst == nil {
t.Fatalf("failed with seed %d: dst is nil.", seed)
}
if !isPubKeyEqual(decrypted.Dst, &key.PublicKey) {
if !IsPubKeyEqual(decrypted.Dst, &key.PublicKey) {
t.Fatalf("failed with seed %d: Dst.", seed)
}
}
}
func TestEncryptWithZeroKey(t *testing.T) {
InitSingleTest()
params, err := generateMessageParams()
if err != nil {
t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err)
}
msg := NewSentMessage(params)
params.KeySym = make([]byte, aesKeyLength)
_, err = msg.Wrap(params)
if err == nil {
t.Fatalf("wrapped with zero key, seed: %d.", seed)
}
params.KeySym = make([]byte, 0)
_, err = msg.Wrap(params)
if err == nil {
t.Fatalf("wrapped with empty key, seed: %d.", seed)
}
params.KeySym = nil
_, err = msg.Wrap(params)
if err == nil {
t.Fatalf("wrapped with nil key, seed: %d.", seed)
}
}

2
whisper/whisperv5/peer_test.go
View file

@ -118,6 +118,7 @@ func initialize(t *testing.T) {
var node TestNode
node.shh = NewWhisper(nil)
node.shh.test = true
node.shh.Start(nil)
topics := make([]TopicType, 0)
topics = append(topics, sharedTopic)
f := Filter{KeySym: sharedKey, Topics: topics}
@ -166,6 +167,7 @@ func stopServers() {
n := nodes[i]
if n != nil {
n.shh.Unwatch(n.filerId)
n.shh.Stop()
n.server.Stop()
}
}

3
whisper/whisperv5/topic.go
View file

@ -14,8 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Contains the Whisper protocol Topic element. For formal details please see
// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#topics.
// Contains the Whisper protocol Topic element.
package whisperv5

105
whisper/whisperv5/whisper.go
View file

@ -22,6 +22,7 @@ import (
crand "crypto/rand"
"crypto/sha256"
"fmt"
"runtime"
"sync"
"time"
@ -45,7 +46,7 @@ type Whisper struct {
symKeys map[string][]byte
keyMu sync.RWMutex
envelopes map[common.Hash]*Envelope // Pool of messages currently tracked by this node
envelopes map[common.Hash]*Envelope // Pool of envelopes currently tracked by this node
messages map[common.Hash]*ReceivedMessage // Pool of successfully decrypted messages, which are not expired yet
expirations map[uint32]*set.SetNonTS // Message expiration pool
poolMu sync.RWMutex // Mutex to sync the message and expiration pools
@ -55,22 +56,28 @@ type Whisper struct {
mailServer MailServer
quit chan struct{}
test bool
messageQueue chan *Envelope
p2pMsgQueue chan *Envelope
quit chan struct{}
overflow bool
test bool
}
// New creates a Whisper client ready to communicate through the Ethereum P2P network.
// Param s should be passed if you want to implement mail server, otherwise nil.
func NewWhisper(server MailServer) *Whisper {
whisper := &Whisper{
privateKeys: make(map[string]*ecdsa.PrivateKey),
symKeys: make(map[string][]byte),
envelopes: make(map[common.Hash]*Envelope),
messages: make(map[common.Hash]*ReceivedMessage),
expirations: make(map[uint32]*set.SetNonTS),
peers: make(map[*Peer]struct{}),
mailServer: server,
quit: make(chan struct{}),
privateKeys: make(map[string]*ecdsa.PrivateKey),
symKeys: make(map[string][]byte),
envelopes: make(map[common.Hash]*Envelope),
messages: make(map[common.Hash]*ReceivedMessage),
expirations: make(map[uint32]*set.SetNonTS),
peers: make(map[*Peer]struct{}),
mailServer: server,
messageQueue: make(chan *Envelope, messageQueueLimit),
p2pMsgQueue: make(chan *Envelope, messageQueueLimit),
quit: make(chan struct{}),
}
whisper.filters = NewFilters(whisper)
@ -124,7 +131,7 @@ func (w *Whisper) RequestHistoricMessages(peerID []byte, data []byte) error {
return err
}
p.trusted = true
return p2p.Send(p.ws, mailRequestCode, data)
return p2p.Send(p.ws, p2pRequestCode, data)
}
func (w *Whisper) SendP2PMessage(peerID []byte, envelope *Envelope) error {
@ -270,6 +277,12 @@ func (w *Whisper) Send(envelope *Envelope) error {
func (w *Whisper) Start(*p2p.Server) error {
glog.V(logger.Info).Infoln("Whisper started")
go w.update()
numCPU := runtime.NumCPU()
for i := 0; i < numCPU; i++ {
go w.processQueue()
}
return nil
}
@ -350,10 +363,10 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
return fmt.Errorf("garbage received (directMessage)")
}
for _, envelope := range envelopes {
wh.postEvent(envelope, p2pCode)
wh.postEvent(envelope, true)
}
}
case mailRequestCode:
case p2pRequestCode:
// Must be processed if mail server is implemented. Otherwise ignore.
if wh.mailServer != nil {
s := rlp.NewStream(packet.Payload, uint64(packet.Size))
@ -382,7 +395,7 @@ func (wh *Whisper) add(envelope *Envelope) error {
if sent > now {
if sent-SynchAllowance > now {
return fmt.Errorf("message created in the future")
return fmt.Errorf("envelope created in the future [%x]", envelope.Hash())
} else {
// recalculate PoW, adjusted for the time difference, plus one second for latency
envelope.calculatePoW(sent - now + 1)
@ -393,30 +406,31 @@ func (wh *Whisper) add(envelope *Envelope) error {
if envelope.Expiry+SynchAllowance*2 < now {
return fmt.Errorf("very old message")
} else {
glog.V(logger.Debug).Infof("expired envelope dropped [%x]", envelope.Hash())
return nil // drop envelope without error
}
}
if len(envelope.Data) > MaxMessageLength {
return fmt.Errorf("huge messages are not allowed")
return fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash())
}
if len(envelope.Version) > 4 {
return fmt.Errorf("oversized Version")
return fmt.Errorf("oversized version [%x]", envelope.Hash())
}
if len(envelope.AESNonce) > AESNonceMaxLength {
// the standard AES GSM nonce size is 12,
// but const gcmStandardNonceSize cannot be accessed directly
return fmt.Errorf("oversized AESNonce")
return fmt.Errorf("oversized AESNonce [%x]", envelope.Hash())
}
if len(envelope.Salt) > saltLength {
return fmt.Errorf("oversized Salt")
return fmt.Errorf("oversized salt [%x]", envelope.Hash())
}
if envelope.PoW() < MinimumPoW && !wh.test {
glog.V(logger.Debug).Infof("envelope with low PoW dropped: %f", envelope.PoW())
glog.V(logger.Debug).Infof("envelope with low PoW dropped: %f [%x]", envelope.PoW(), envelope.Hash())
return nil // drop envelope without error
}
@ -436,22 +450,59 @@ func (wh *Whisper) add(envelope *Envelope) error {
wh.poolMu.Unlock()
if alreadyCached {
glog.V(logger.Detail).Infof("whisper envelope already cached: %x\n", envelope)
glog.V(logger.Detail).Infof("whisper envelope already cached [%x]\n", envelope.Hash())
} else {
wh.postEvent(envelope, messagesCode) // notify the local node about the new message
glog.V(logger.Detail).Infof("cached whisper envelope %v\n", envelope)
glog.V(logger.Detail).Infof("cached whisper envelope [%x]: %v\n", envelope.Hash(), envelope)
wh.postEvent(envelope, false) // notify the local node about the new message
}
return nil
}
// postEvent delivers the message to the watchers.
func (w *Whisper) postEvent(envelope *Envelope, messageCode uint64) {
// postEvent queues the message for further processing.
func (w *Whisper) postEvent(envelope *Envelope, isP2P bool) {
// if the version of incoming message is higher than
// currently supported version, we can not decrypt it,
// and therefore just ignore this message
if envelope.Ver() <= EnvelopeVersion {
// todo: review if you need an additional thread here
go w.filters.NotifyWatchers(envelope, messageCode)
if isP2P {
w.p2pMsgQueue <- envelope
} else {
w.checkOverflow()
w.messageQueue <- envelope
}
}
}
// checkOverflow checks if message queue overflow occurs and reports it if necessary.
func (w *Whisper) checkOverflow() {
queueSize := len(w.messageQueue)
if queueSize == messageQueueLimit {
if !w.overflow {
w.overflow = true
glog.V(logger.Warn).Infoln("message queue overflow")
}
} else if queueSize <= messageQueueLimit/2 {
if w.overflow {
w.overflow = false
}
}
}
// processQueue delivers the messages to the watchers during the lifetime of the whisper node.
func (w *Whisper) processQueue() {
var e *Envelope
for {
select {
case <-w.quit:
return
case e = <-w.messageQueue:
w.filters.NotifyWatchers(e, false)
case e = <-w.p2pMsgQueue:
w.filters.NotifyWatchers(e, true)
}
}
}

54
whisper/whisperv5/whisper_test.go
View file

@ -19,6 +19,7 @@ package whisperv5
import (
"bytes"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
@ -309,3 +310,56 @@ func TestWhisperSymKeyManagement(t *testing.T) {
t.Fatalf("failed to delete second key: second key is not nil.")
}
}
func TestExpiry(t *testing.T) {
InitSingleTest()
w := NewWhisper(nil)
w.test = true
w.Start(nil)
defer w.Stop()
params, err := generateMessageParams()
if err != nil {
t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err)
}
params.TTL = 1
msg := NewSentMessage(params)
env, err := msg.Wrap(params)
if err != nil {
t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
}
err = w.Send(env)
if err != nil {
t.Fatalf("failed to send envelope with seed %d: %s.", seed, err)
}
// wait till received or timeout
var received, expired bool
for j := 0; j < 20; j++ {
time.Sleep(100 * time.Millisecond)
if len(w.Envelopes()) > 0 {
received = true
break
}
}
if !received {
t.Fatalf("did not receive the sent envelope, seed: %d.", seed)
}
// wait till expired or timeout
for j := 0; j < 20; j++ {
time.Sleep(100 * time.Millisecond)
if len(w.Envelopes()) == 0 {
expired = true
break
}
}
if !expired {
t.Fatalf("expire failed, seed: %d.", seed)
}
}