diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0a09baef7d..9a61d39325 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,24 +9,27 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi +p2p/simulations @lmars +p2p/protocols @zelig +swarm/api/http @justelad swarm/bmt @zelig swarm/dev @lmars swarm/fuse @jmozah @holisticode swarm/grafana_dashboards @nonsense swarm/metrics @nonsense @holisticode swarm/multihash @nolash -swarm/network/bitvector @zelig @janos @gbalint -swarm/network/priorityqueue @zelig @janos @gbalint -swarm/network/simulations @zelig -swarm/network/stream @janos @zelig @gbalint @holisticode @justelad +swarm/network/bitvector @zelig @janos +swarm/network/priorityqueue @zelig @janos +swarm/network/simulations @zelig @janos +swarm/network/stream @janos @zelig @holisticode @justelad swarm/network/stream/intervals @janos swarm/network/stream/testing @zelig swarm/pot @zelig swarm/pss @nolash @zelig @nonsense swarm/services @zelig swarm/state @justelad -swarm/storage/encryption @gbalint @zelig @nagydani +swarm/storage/encryption @zelig @nagydani swarm/storage/mock @janos -swarm/storage/mru @nolash +swarm/storage/feed @nolash @jpeletier swarm/testutil @lmars whisper/ @gballet @gluk256 diff --git a/.travis.yml b/.travis.yml index 372f7a8270..c1cc7c4aa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum sudo: false matrix: include: - - os: linux - dist: trusty - sudo: required - go: 1.9.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage $TEST_PACKAGES - - os: linux dist: trusty sudo: required @@ -56,7 +45,8 @@ matrix: - go run build/ci.go lint # This builder does the Ubuntu PPA upload - - os: linux + - if: type = push + os: linux dist: trusty go: 1.11.x env: @@ -74,7 +64,8 @@ matrix: - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder " -upload ppa:ethereum/ethereum # This builder does the Linux Azure uploads - - os: linux + - if: type = push + os: linux dist: trusty sudo: required go: 1.11.x @@ -107,7 +98,8 @@ matrix: - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds # This builder does the Linux Azure MIPS xgo uploads - - os: linux + - if: type = push + os: linux dist: trusty services: - docker @@ -134,7 +126,8 @@ matrix: - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds # This builder does the Android Maven and Azure uploads - - os: linux + - if: type = push + os: linux dist: trusty addons: apt: @@ -155,7 +148,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.11.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go @@ -171,7 +164,8 @@ matrix: - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads - - os: osx + - if: type = push + os: osx go: 1.11.x env: - azure-osx @@ -199,7 +193,8 @@ matrix: - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds # This builder does the Azure archive purges to avoid accumulating junk - - os: linux + - if: type = cron + os: linux dist: trusty go: 1.11.x env: @@ -208,10 +203,3 @@ matrix: submodules: false # avoid cloning ethereum/tests script: - go run build/ci.go purge -store gethstore/builds -days 14 - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/e09ccdce1048c5e03445 - on_success: change - on_failure: always diff --git a/Makefile b/Makefile index 0c1bb3bce7..966bf9cbba 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,9 @@ devtools: @type "solc" 2> /dev/null || echo 'Please install solc' @type "protoc" 2> /dev/null || echo 'Please install protoc' +swarm-devtools: + env GOBIN= go install ./cmd/swarm/mimegen + # Cross Compilation Targets (xgo) geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 254b1f7fb4..535e5d78b2 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -137,6 +137,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { // MethodById looks up a method by the 4-byte id // returns nil if none found func (abi *ABI) MethodById(sigdata []byte) (*Method, error) { + if len(sigdata) < 4 { + return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata)) + } for _, method := range abi.Methods { if bytes.Equal(method.Id(), sigdata[:4]) { return &method, nil diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 8018df7756..59ba79cb6d 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -711,5 +711,14 @@ func TestABI_MethodById(t *testing.T) { t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id())) } } - + // Also test empty + if _, err := abi.MethodById([]byte{0x00}); err == nil { + t.Errorf("Expected error, too short to decode data") + } + if _, err := abi.MethodById([]byte{}); err == nil { + t.Errorf("Expected error, too short to decode data") + } + if _, err := abi.MethodById(nil); err == nil { + t.Errorf("Expected error, nil is short to decode data") + } } diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index fc0ccbf52c..6f46fc1495 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -208,7 +208,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad } // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated -// chain doens't have miners, we just return a gas price of 1 for any call. +// chain doesn't have miners, we just return a gas price of 1 for any call. func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { return big.NewInt(1), nil } diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 172bed8fa0..4dca4b4ea3 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -23,13 +23,13 @@ package bind import ( "bytes" "fmt" + "go/format" "regexp" "strings" "text/template" "unicode" "github.com/ethereum/go-ethereum/accounts/abi" - "golang.org/x/tools/imports" ) // Lang is a target programming language selector to generate bindings for. @@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La if err := tmpl.Execute(buffer, data); err != nil { return "", err } - // For Go bindings pass the code through goimports to clean it up and double check + // For Go bindings pass the code through gofmt to clean it up if lang == LangGo { - code, err := imports.Process(".", buffer.Bytes(), nil) + code, err := format.Source(buffer.Bytes()) if err != nil { return "", fmt.Errorf("%v\n%s", err, buffer) } diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 0e5b1c1616..46e0b38d00 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -27,7 +27,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "golang.org/x/tools/imports" ) var bindTests = []struct { @@ -35,6 +34,7 @@ var bindTests = []struct { contract string bytecode string abi string + imports string tester string }{ // Test that the binding is available in combined and separate forms too @@ -43,6 +43,7 @@ var bindTests = []struct { `contract NilContract {}`, `606060405260068060106000396000f3606060405200`, `[]`, + `"github.com/ethereum/go-ethereum/common"`, ` if b, err := NewEmpty(common.Address{}, nil); b == nil || err != nil { t.Fatalf("combined binding (%v) nil or error (%v) not nil", b, nil) @@ -61,6 +62,7 @@ var bindTests = []struct { `https://ethereum.org/token`, `60606040526040516107fd3803806107fd83398101604052805160805160a05160c051929391820192909101600160a060020a0333166000908152600360209081526040822086905581548551838052601f6002600019610100600186161502019093169290920482018390047f290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e56390810193919290918801908390106100e857805160ff19168380011785555b506101189291505b8082111561017157600081556001016100b4565b50506002805460ff19168317905550505050610658806101a56000396000f35b828001600101855582156100ac579182015b828111156100ac5782518260005055916020019190600101906100fa565b50508060016000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f1061017557805160ff19168380011785555b506100c89291506100b4565b5090565b82800160010185558215610165579182015b8281111561016557825182600050559160200191906001019061018756606060405236156100775760e060020a600035046306fdde03811461007f57806323b872dd146100dc578063313ce5671461010e57806370a082311461011a57806395d89b4114610132578063a9059cbb1461018e578063cae9ca51146101bd578063dc3080f21461031c578063dd62ed3e14610341575b610365610002565b61036760008054602060026001831615610100026000190190921691909104601f810182900490910260809081016040526060828152929190828280156104eb5780601f106104c0576101008083540402835291602001916104eb565b6103d5600435602435604435600160a060020a038316600090815260036020526040812054829010156104f357610002565b6103e760025460ff1681565b6103d560043560036020526000908152604090205481565b610367600180546020600282841615610100026000190190921691909104601f810182900490910260809081016040526060828152929190828280156104eb5780601f106104c0576101008083540402835291602001916104eb565b610365600435602435600160a060020a033316600090815260036020526040902054819010156103f157610002565b60806020604435600481810135601f8101849004909302840160405260608381526103d5948235946024803595606494939101919081908382808284375094965050505050505060006000836004600050600033600160a060020a03168152602001908152602001600020600050600087600160a060020a031681526020019081526020016000206000508190555084905080600160a060020a0316638f4ffcb1338630876040518560e060020a0281526004018085600160a060020a0316815260200184815260200183600160a060020a03168152602001806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600f02600301f150905090810190601f1680156102f25780820380516001836020036101000a031916815260200191505b50955050505050506000604051808303816000876161da5a03f11561000257505050509392505050565b6005602090815260043560009081526040808220909252602435815220546103d59081565b60046020818152903560009081526040808220909252602435815220546103d59081565b005b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600f02600301f150905090810190601f1680156103c75780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60408051918252519081900360200190f35b6060908152602090f35b600160a060020a03821660009081526040902054808201101561041357610002565b806003600050600033600160a060020a03168152602001908152602001600020600082828250540392505081905550806003600050600084600160a060020a0316815260200190815260200160002060008282825054019250508190555081600160a060020a031633600160a060020a03167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef836040518082815260200191505060405180910390a35050565b820191906000526020600020905b8154815290600101906020018083116104ce57829003601f168201915b505050505081565b600160a060020a03831681526040812054808301101561051257610002565b600160a060020a0380851680835260046020908152604080852033949094168086529382528085205492855260058252808520938552929052908220548301111561055c57610002565b816003600050600086600160a060020a03168152602001908152602001600020600082828250540392505081905550816003600050600085600160a060020a03168152602001908152602001600020600082828250540192505081905550816005600050600086600160a060020a03168152602001908152602001600020600050600033600160a060020a0316815260200190815260200160002060008282825054019250508190555082600160a060020a031633600160a060020a03167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a3939250505056`, `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"success","type":"bool"}],"type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint8"}],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"},{"name":"_extraData","type":"bytes"}],"name":"approveAndCall","outputs":[{"name":"success","type":"bool"}],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"},{"name":"","type":"address"}],"name":"spentAllowance","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"},{"name":"","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"inputs":[{"name":"initialSupply","type":"uint256"},{"name":"tokenName","type":"string"},{"name":"decimalUnits","type":"uint8"},{"name":"tokenSymbol","type":"string"}],"type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"}]`, + `"github.com/ethereum/go-ethereum/common"`, ` if b, err := NewToken(common.Address{}, nil); b == nil || err != nil { t.Fatalf("binding (%v) nil or error (%v) not nil", b, nil) @@ -72,6 +74,7 @@ var bindTests = []struct { `https://ethereum.org/crowdsale`, `606060408190526007805460ff1916905560a0806105a883396101006040529051608051915160c05160e05160008054600160a060020a03199081169095178155670de0b6b3a7640000958602600155603c9093024201600355930260045560058054909216909217905561052f90819061007990396000f36060604052361561006c5760e060020a600035046301cb3b20811461008257806329dcb0cf1461014457806338af3eed1461014d5780636e66f6e91461015f5780637a3a0e84146101715780637b3e5e7b1461017a578063a035b1fe14610183578063dc0d3dff1461018c575b61020060075460009060ff161561032357610002565b61020060035460009042106103205760025460015490106103cb576002548154600160a060020a0316908290606082818181858883f150915460025460408051600160a060020a039390931683526020830191909152818101869052517fe842aea7a5f1b01049d752008c53c52890b1a6daf660cf39e8eec506112bbdf6945090819003909201919050a15b60405160008054600160a060020a039081169230909116319082818181858883f150506007805460ff1916600117905550505050565b6103a160035481565b6103ab600054600160a060020a031681565b6103ab600554600160a060020a031681565b6103a160015481565b6103a160025481565b6103a160045481565b6103be60043560068054829081101561000257506000526002027ff652222313e28459528d920b65115c16c04f3efc82aaedc97be59f3f377c0d3f8101547ff652222313e28459528d920b65115c16c04f3efc82aaedc97be59f3f377c0d409190910154600160a060020a03919091169082565b005b505050815481101561000257906000526020600020906002020160005060008201518160000160006101000a815481600160a060020a030219169083021790555060208201518160010160005055905050806002600082828250540192505081905550600560009054906101000a9004600160a060020a0316600160a060020a031663a9059cbb3360046000505484046040518360e060020a0281526004018083600160a060020a03168152602001828152602001925050506000604051808303816000876161da5a03f11561000257505060408051600160a060020a03331681526020810184905260018183015290517fe842aea7a5f1b01049d752008c53c52890b1a6daf660cf39e8eec506112bbdf692509081900360600190a15b50565b5060a0604052336060908152346080819052600680546001810180835592939282908280158290116102025760020281600202836000526020600020918201910161020291905b8082111561039d57805473ffffffffffffffffffffffffffffffffffffffff19168155600060019190910190815561036a565b5090565b6060908152602090f35b600160a060020a03166060908152602090f35b6060918252608052604090f35b5b60065481101561010e576006805482908110156100025760009182526002027ff652222313e28459528d920b65115c16c04f3efc82aaedc97be59f3f377c0d3f0190600680549254600160a060020a0316928490811015610002576002027ff652222313e28459528d920b65115c16c04f3efc82aaedc97be59f3f377c0d40015460405190915082818181858883f19350505050507fe842aea7a5f1b01049d752008c53c52890b1a6daf660cf39e8eec506112bbdf660066000508281548110156100025760008290526002027ff652222313e28459528d920b65115c16c04f3efc82aaedc97be59f3f377c0d3f01548154600160a060020a039190911691908490811015610002576002027ff652222313e28459528d920b65115c16c04f3efc82aaedc97be59f3f377c0d40015460408051600160a060020a0394909416845260208401919091526000838201525191829003606001919050a16001016103cc56`, `[{"constant":false,"inputs":[],"name":"checkGoalReached","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"deadline","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"beneficiary","outputs":[{"name":"","type":"address"}],"type":"function"},{"constant":true,"inputs":[],"name":"tokenReward","outputs":[{"name":"","type":"address"}],"type":"function"},{"constant":true,"inputs":[],"name":"fundingGoal","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"amountRaised","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"price","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"funders","outputs":[{"name":"addr","type":"address"},{"name":"amount","type":"uint256"}],"type":"function"},{"inputs":[{"name":"ifSuccessfulSendTo","type":"address"},{"name":"fundingGoalInEthers","type":"uint256"},{"name":"durationInMinutes","type":"uint256"},{"name":"etherCostOfEachToken","type":"uint256"},{"name":"addressOfTokenUsedAsReward","type":"address"}],"type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"name":"backer","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"isContribution","type":"bool"}],"name":"FundTransfer","type":"event"}]`, + `"github.com/ethereum/go-ethereum/common"`, ` if b, err := NewCrowdsale(common.Address{}, nil); b == nil || err != nil { t.Fatalf("binding (%v) nil or error (%v) not nil", b, nil) @@ -83,6 +86,7 @@ var bindTests = []struct { `https://ethereum.org/dao`, `606060405260405160808061145f833960e06040529051905160a05160c05160008054600160a060020a03191633179055600184815560028490556003839055600780549182018082558280158290116100b8576003028160030283600052602060002091820191016100b891906101c8565b50506060919091015160029190910155600160a060020a0381166000146100a65760008054600160a060020a031916821790555b505050506111f18061026e6000396000f35b505060408051608081018252600080825260208281018290528351908101845281815292820192909252426060820152600780549194509250811015610002579081527fa66cc928b5edb82af9bd49922954155ab7b0942694bea4ce44661d9a8736c6889050815181546020848101517401000000000000000000000000000000000000000002600160a060020a03199290921690921760a060020a60ff021916178255604083015180516001848101805460008281528690209195600293821615610100026000190190911692909204601f9081018390048201949192919091019083901061023e57805160ff19168380011785555b50610072929150610226565b5050600060028201556001015b8082111561023a578054600160a860020a031916815560018181018054600080835592600290821615610100026000190190911604601f81901061020c57506101bb565b601f0160209004906000526020600020908101906101bb91905b8082111561023a5760008155600101610226565b5090565b828001600101855582156101af579182015b828111156101af57825182600050559160200191906001019061025056606060405236156100b95760e060020a6000350463013cf08b81146100bb578063237e9492146101285780633910682114610281578063400e3949146102995780635daf08ca146102a257806369bd34361461032f5780638160f0b5146103385780638da5cb5b146103415780639644fcbd14610353578063aa02a90f146103be578063b1050da5146103c7578063bcca1fd3146104b5578063d3c0715b146104dc578063eceb29451461058d578063f2fde38b1461067b575b005b61069c6004356004805482908110156100025790600052602060002090600a02016000506005810154815460018301546003840154600485015460068601546007870154600160a060020a03959095169750929560020194919360ff828116946101009093041692919089565b60408051602060248035600481810135601f81018590048502860185019096528585526107759581359591946044949293909201918190840183828082843750949650505050505050600060006004600050848154811015610002575090527f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19e600a8402908101547f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19b909101904210806101e65750600481015460ff165b8061026757508060000160009054906101000a9004600160a060020a03168160010160005054846040518084600160a060020a0316606060020a0281526014018381526020018280519060200190808383829060006004602084601f0104600f02600301f15090500193505050506040518091039020816007016000505414155b8061027757506001546005820154105b1561109257610002565b61077560043560066020526000908152604090205481565b61077560055481565b61078760043560078054829081101561000257506000526003026000805160206111d18339815191528101547fa66cc928b5edb82af9bd49922954155ab7b0942694bea4ce44661d9a8736c68a820154600160a060020a0382169260a060020a90920460ff16917fa66cc928b5edb82af9bd49922954155ab7b0942694bea4ce44661d9a8736c689019084565b61077560025481565b61077560015481565b610830600054600160a060020a031681565b604080516020604435600481810135601f81018490048402850184019095528484526100b9948135946024803595939460649492939101918190840183828082843750949650505050505050600080548190600160a060020a03908116339091161461084d57610002565b61077560035481565b604080516020604435600481810135601f8101849004840285018401909552848452610775948135946024803595939460649492939101918190840183828082843750506040805160209735808a0135601f81018a90048a0283018a019093528282529698976084979196506024909101945090925082915084018382808284375094965050505050505033600160a060020a031660009081526006602052604081205481908114806104ab5750604081205460078054909190811015610002579082526003026000805160206111d1833981519152015460a060020a900460ff16155b15610ce557610002565b6100b960043560243560443560005433600160a060020a03908116911614610b1857610002565b604080516020604435600481810135601f810184900484028501840190955284845261077594813594602480359593946064949293910191819084018382808284375094965050505050505033600160a060020a031660009081526006602052604081205481908114806105835750604081205460078054909190811015610002579082526003026000805160206111d18339815191520181505460a060020a900460ff16155b15610f1d57610002565b604080516020606435600481810135601f81018490048402850184019095528484526107759481359460248035956044359560849492019190819084018382808284375094965050505050505060006000600460005086815481101561000257908252600a027f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19b01815090508484846040518084600160a060020a0316606060020a0281526014018381526020018280519060200190808383829060006004602084601f0104600f02600301f150905001935050505060405180910390208160070160005054149150610cdc565b6100b960043560005433600160a060020a03908116911614610f0857610002565b604051808a600160a060020a031681526020018981526020018060200188815260200187815260200186815260200185815260200184815260200183815260200182810382528981815460018160011615610100020316600290048152602001915080546001816001161561010002031660029004801561075e5780601f106107335761010080835404028352916020019161075e565b820191906000526020600020905b81548152906001019060200180831161074157829003601f168201915b50509a505050505050505050505060405180910390f35b60408051918252519081900360200190f35b60408051600160a060020a038616815260208101859052606081018390526080918101828152845460026001821615610100026000190190911604928201839052909160a08301908590801561081e5780601f106107f35761010080835404028352916020019161081e565b820191906000526020600020905b81548152906001019060200180831161080157829003601f168201915b50509550505050505060405180910390f35b60408051600160a060020a03929092168252519081900360200190f35b600160a060020a03851660009081526006602052604081205414156108a957604060002060078054918290556001820180825582801582901161095c5760030281600302836000526020600020918201910161095c9190610a4f565b600160a060020a03851660009081526006602052604090205460078054919350908390811015610002575060005250600381026000805160206111d183398151915201805474ff0000000000000000000000000000000000000000191660a060020a85021781555b60408051600160a060020a03871681526020810186905281517f27b022af4a8347100c7a041ce5ccf8e14d644ff05de696315196faae8cd50c9b929181900390910190a15050505050565b505050915081506080604051908101604052808681526020018581526020018481526020014281526020015060076000508381548110156100025790600052602060002090600302016000508151815460208481015160a060020a02600160a060020a03199290921690921774ff00000000000000000000000000000000000000001916178255604083015180516001848101805460008281528690209195600293821615610100026000190190911692909204601f90810183900482019491929190910190839010610ad357805160ff19168380011785555b50610b03929150610abb565b5050600060028201556001015b80821115610acf57805474ffffffffffffffffffffffffffffffffffffffffff1916815560018181018054600080835592600290821615610100026000190190911604601f819010610aa15750610a42565b601f016020900490600052602060002090810190610a4291905b80821115610acf5760008155600101610abb565b5090565b82800160010185558215610a36579182015b82811115610a36578251826000505591602001919060010190610ae5565b50506060919091015160029190910155610911565b600183905560028290556003819055604080518481526020810184905280820183905290517fa439d3fa452be5e0e1e24a8145e715f4fd8b9c08c96a42fd82a855a85e5d57de9181900360600190a1505050565b50508585846040518084600160a060020a0316606060020a0281526014018381526020018280519060200190808383829060006004602084601f0104600f02600301f150905001935050505060405180910390208160070160005081905550600260005054603c024201816003016000508190555060008160040160006101000a81548160ff0219169083021790555060008160040160016101000a81548160ff02191690830217905550600081600501600050819055507f646fec02522b41e7125cfc859a64fd4f4cefd5dc3b6237ca0abe251ded1fa881828787876040518085815260200184600160a060020a03168152602001838152602001806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600f02600301f150905090810190601f168015610cc45780820380516001836020036101000a031916815260200191505b509550505050505060405180910390a1600182016005555b50949350505050565b6004805460018101808355909190828015829011610d1c57600a0281600a028360005260206000209182019101610d1c9190610db8565b505060048054929450918491508110156100025790600052602060002090600a02016000508054600160a060020a031916871781556001818101879055855160028381018054600082815260209081902096975091959481161561010002600019011691909104601f90810182900484019391890190839010610ed857805160ff19168380011785555b50610b6c929150610abb565b50506001015b80821115610acf578054600160a060020a03191681556000600182810182905560028381018054848255909281161561010002600019011604601f819010610e9c57505b5060006003830181905560048301805461ffff191690556005830181905560068301819055600783018190556008830180548282559082526020909120610db2916002028101905b80821115610acf57805474ffffffffffffffffffffffffffffffffffffffffff1916815560018181018054600080835592600290821615610100026000190190911604601f819010610eba57505b5050600101610e44565b601f016020900490600052602060002090810190610dfc9190610abb565b601f016020900490600052602060002090810190610e929190610abb565b82800160010185558215610da6579182015b82811115610da6578251826000505591602001919060010190610eea565b60008054600160a060020a0319168217905550565b600480548690811015610002576000918252600a027f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19b01905033600160a060020a0316600090815260098201602052604090205490915060ff1660011415610f8457610002565b33600160a060020a031660009081526009820160205260409020805460ff1916600190811790915560058201805490910190558315610fcd576006810180546001019055610fda565b6006810180546000190190555b7fc34f869b7ff431b034b7b9aea9822dac189a685e0b015c7d1be3add3f89128e8858533866040518085815260200184815260200183600160a060020a03168152602001806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600f02600301f150905090810190601f16801561107a5780820380516001836020036101000a031916815260200191505b509550505050505060405180910390a1509392505050565b6006810154600354901315611158578060000160009054906101000a9004600160a060020a0316600160a060020a03168160010160005054670de0b6b3a76400000284604051808280519060200190808383829060006004602084601f0104600f02600301f150905090810190601f1680156111225780820380516001836020036101000a031916815260200191505b5091505060006040518083038185876185025a03f15050505060048101805460ff191660011761ff00191661010017905561116d565b60048101805460ff191660011761ff00191690555b60068101546005820154600483015460408051888152602081019490945283810192909252610100900460ff166060830152517fd220b7272a8b6d0d7d6bcdace67b936a8f175e6d5c1b3ee438b72256b32ab3af9181900360800190a1509291505056a66cc928b5edb82af9bd49922954155ab7b0942694bea4ce44661d9a8736c688`, `[{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"proposals","outputs":[{"name":"recipient","type":"address"},{"name":"amount","type":"uint256"},{"name":"description","type":"string"},{"name":"votingDeadline","type":"uint256"},{"name":"executed","type":"bool"},{"name":"proposalPassed","type":"bool"},{"name":"numberOfVotes","type":"uint256"},{"name":"currentResult","type":"int256"},{"name":"proposalHash","type":"bytes32"}],"type":"function"},{"constant":false,"inputs":[{"name":"proposalNumber","type":"uint256"},{"name":"transactionBytecode","type":"bytes"}],"name":"executeProposal","outputs":[{"name":"result","type":"int256"}],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"memberId","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"numProposals","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"members","outputs":[{"name":"member","type":"address"},{"name":"canVote","type":"bool"},{"name":"name","type":"string"},{"name":"memberSince","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"debatingPeriodInMinutes","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"minimumQuorum","outputs":[{"name":"","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"type":"function"},{"constant":false,"inputs":[{"name":"targetMember","type":"address"},{"name":"canVote","type":"bool"},{"name":"memberName","type":"string"}],"name":"changeMembership","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"majorityMargin","outputs":[{"name":"","type":"int256"}],"type":"function"},{"constant":false,"inputs":[{"name":"beneficiary","type":"address"},{"name":"etherAmount","type":"uint256"},{"name":"JobDescription","type":"string"},{"name":"transactionBytecode","type":"bytes"}],"name":"newProposal","outputs":[{"name":"proposalID","type":"uint256"}],"type":"function"},{"constant":false,"inputs":[{"name":"minimumQuorumForProposals","type":"uint256"},{"name":"minutesForDebate","type":"uint256"},{"name":"marginOfVotesForMajority","type":"int256"}],"name":"changeVotingRules","outputs":[],"type":"function"},{"constant":false,"inputs":[{"name":"proposalNumber","type":"uint256"},{"name":"supportsProposal","type":"bool"},{"name":"justificationText","type":"string"}],"name":"vote","outputs":[{"name":"voteID","type":"uint256"}],"type":"function"},{"constant":true,"inputs":[{"name":"proposalNumber","type":"uint256"},{"name":"beneficiary","type":"address"},{"name":"etherAmount","type":"uint256"},{"name":"transactionBytecode","type":"bytes"}],"name":"checkProposalCode","outputs":[{"name":"codeChecksOut","type":"bool"}],"type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"type":"function"},{"inputs":[{"name":"minimumQuorumForProposals","type":"uint256"},{"name":"minutesForDebate","type":"uint256"},{"name":"marginOfVotesForMajority","type":"int256"},{"name":"congressLeader","type":"address"}],"type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"name":"proposalID","type":"uint256"},{"indexed":false,"name":"recipient","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"description","type":"string"}],"name":"ProposalAdded","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"proposalID","type":"uint256"},{"indexed":false,"name":"position","type":"bool"},{"indexed":false,"name":"voter","type":"address"},{"indexed":false,"name":"justification","type":"string"}],"name":"Voted","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"proposalID","type":"uint256"},{"indexed":false,"name":"result","type":"int256"},{"indexed":false,"name":"quorum","type":"uint256"},{"indexed":false,"name":"active","type":"bool"}],"name":"ProposalTallied","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"member","type":"address"},{"indexed":false,"name":"isMember","type":"bool"}],"name":"MembershipChanged","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"minimumQuorum","type":"uint256"},{"indexed":false,"name":"debatingPeriodInMinutes","type":"uint256"},{"indexed":false,"name":"majorityMargin","type":"int256"}],"name":"ChangeOfRules","type":"event"}]`, + `"github.com/ethereum/go-ethereum/common"`, ` if b, err := NewDAO(common.Address{}, nil); b == nil || err != nil { t.Fatalf("binding (%v) nil or error (%v) not nil", b, nil) @@ -102,6 +106,11 @@ var bindTests = []struct { {"type":"function","name":"mixedInputs","constant":true,"inputs":[{"name":"","type":"string"},{"name":"str","type":"string"}],"outputs":[]} ] `, + ` + "fmt" + + "github.com/ethereum/go-ethereum/common" + `, `if b, err := NewInputChecker(common.Address{}, nil); b == nil || err != nil { t.Fatalf("binding (%v) nil or error (%v) not nil", b, nil) } else if false { // Don't run, just compile and test types @@ -131,6 +140,11 @@ var bindTests = []struct { {"type":"function","name":"mixedOutputs","constant":true,"inputs":[],"outputs":[{"name":"","type":"string"},{"name":"str","type":"string"}]} ] `, + ` + "fmt" + + "github.com/ethereum/go-ethereum/common" + `, `if b, err := NewOutputChecker(common.Address{}, nil); b == nil || err != nil { t.Fatalf("binding (%v) nil or error (%v) not nil", b, nil) } else if false { // Don't run, just compile and test types @@ -160,6 +174,13 @@ var bindTests = []struct { {"type":"event","name":"dynamic","inputs":[{"name":"idxStr","type":"string","indexed":true},{"name":"idxDat","type":"bytes","indexed":true},{"name":"str","type":"string"},{"name":"dat","type":"bytes"}]} ] `, + ` + "fmt" + "math/big" + "reflect" + + "github.com/ethereum/go-ethereum/common" + `, `if e, err := NewEventChecker(common.Address{}, nil); e == nil || err != nil { t.Fatalf("binding (%v) nil or error (%v) not nil", e, nil) } else if false { // Don't run, just compile and test types @@ -225,6 +246,14 @@ var bindTests = []struct { `, `6060604052604051610328380380610328833981016040528051018060006000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f10608d57805160ff19168380011785555b50607c9291505b8082111560ba57838155600101606b565b50505061026a806100be6000396000f35b828001600101855582156064579182015b828111156064578251826000505591602001919060010190609e565b509056606060405260e060020a60003504630d86a0e181146100315780636874e8091461008d578063d736c513146100ea575b005b610190600180546020600282841615610100026000190190921691909104601f810182900490910260809081016040526060828152929190828280156102295780601f106101fe57610100808354040283529160200191610229565b61019060008054602060026001831615610100026000190190921691909104601f810182900490910260809081016040526060828152929190828280156102295780601f106101fe57610100808354040283529160200191610229565b60206004803580820135601f81018490049093026080908101604052606084815261002f946024939192918401918190838280828437509496505050505050508060016000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f1061023157805160ff19168380011785555b506102619291505b808211156102665760008155830161017d565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600f02600301f150905090810190601f1680156101f05780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b820191906000526020600020905b81548152906001019060200180831161020c57829003601f168201915b505050505081565b82800160010185558215610175579182015b82811115610175578251826000505591602001919060010190610243565b505050565b509056`, `[{"constant":true,"inputs":[],"name":"transactString","outputs":[{"name":"","type":"string"}],"type":"function"},{"constant":true,"inputs":[],"name":"deployString","outputs":[{"name":"","type":"string"}],"type":"function"},{"constant":false,"inputs":[{"name":"str","type":"string"}],"name":"transact","outputs":[],"type":"function"},{"inputs":[{"name":"str","type":"string"}],"type":"constructor"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -266,6 +295,14 @@ var bindTests = []struct { `, `606060405260dc8060106000396000f3606060405260e060020a6000350463993a04b78114601a575b005b600060605260c0604052600260809081527f486900000000000000000000000000000000000000000000000000000000000060a05260017fc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a47060e0829052610100819052606060c0908152600261012081905281906101409060a09080838184600060046012f1505081517fffff000000000000000000000000000000000000000000000000000000000000169091525050604051610160819003945092505050f3`, `[{"constant":true,"inputs":[],"name":"getter","outputs":[{"name":"","type":"string"},{"name":"","type":"int256"},{"name":"","type":"bytes32"}],"type":"function"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -298,6 +335,14 @@ var bindTests = []struct { `, `606060405260dc8060106000396000f3606060405260e060020a60003504633175aae28114601a575b005b600060605260c0604052600260809081527f486900000000000000000000000000000000000000000000000000000000000060a05260017fc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a47060e0829052610100819052606060c0908152600261012081905281906101409060a09080838184600060046012f1505081517fffff000000000000000000000000000000000000000000000000000000000000169091525050604051610160819003945092505050f3`, `[{"constant":true,"inputs":[],"name":"tuple","outputs":[{"name":"a","type":"string"},{"name":"b","type":"int256"},{"name":"c","type":"bytes32"}],"type":"function"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -340,6 +385,16 @@ var bindTests = []struct { `, `606060405261015c806100126000396000f3606060405260e060020a6000350463be1127a3811461003c578063d88becc014610092578063e15a3db71461003c578063f637e5891461003c575b005b604080516020600480358082013583810285810185019096528085526100ee959294602494909392850192829185019084908082843750949650505050505050604080516020810190915260009052805b919050565b604080516102e0818101909252610138916004916102e491839060179083908390808284375090955050505050506102e0604051908101604052806017905b60008152602001906001900390816100d15790505081905061008d565b60405180806020018281038252838181518152602001915080519060200190602002808383829060006004602084601f0104600f02600301f1509050019250505060405180910390f35b60405180826102e0808381846000600461015cf15090500191505060405180910390f3`, `[{"constant":true,"inputs":[{"name":"input","type":"address[]"}],"name":"echoAddresses","outputs":[{"name":"output","type":"address[]"}],"type":"function"},{"constant":true,"inputs":[{"name":"input","type":"uint24[23]"}],"name":"echoFancyInts","outputs":[{"name":"output","type":"uint24[23]"}],"type":"function"},{"constant":true,"inputs":[{"name":"input","type":"int256[]"}],"name":"echoInts","outputs":[{"name":"output","type":"int256[]"}],"type":"function"},{"constant":true,"inputs":[{"name":"input","type":"bool[]"}],"name":"echoBools","outputs":[{"name":"output","type":"bool[]"}],"type":"function"}]`, + ` + "math/big" + "reflect" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -374,6 +429,14 @@ var bindTests = []struct { `, `6060604052606a8060106000396000f360606040523615601d5760e060020a6000350463fc9c8d3981146040575b605e6000805473ffffffffffffffffffffffffffffffffffffffff191633179055565b606060005473ffffffffffffffffffffffffffffffffffffffff1681565b005b6060908152602090f3`, `[{"constant":true,"inputs":[],"name":"caller","outputs":[{"name":"","type":"address"}],"type":"function"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -409,6 +472,11 @@ var bindTests = []struct { `, `6060604052609f8060106000396000f3606060405260e060020a6000350463f97a60058114601a575b005b600060605260c0604052600d60809081527f4920646f6e27742065786973740000000000000000000000000000000000000060a052602060c0908152600d60e081905281906101009060a09080838184600060046012f15050815172ffffffffffffffffffffffffffffffffffffff1916909152505060405161012081900392509050f3`, `[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`, + ` + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + `, ` // Create a simulator and wrap a non-deployed contract sim := backends.NewSimulatedBackend(nil, uint64(10000000000)) @@ -443,6 +511,14 @@ var bindTests = []struct { `, `606060405261021c806100126000396000f3606060405260e060020a600035046323fcf32a81146100265780634f28bf0e1461007b575b005b6040805160206004803580820135601f8101849004840285018401909552848452610024949193602493909291840191908190840183828082843750949650505050505050620186a05a101561014e57610002565b6100db60008054604080516020601f600260001961010060018816150201909516949094049384018190048102820181019092528281529291908301828280156102145780601f106101e957610100808354040283529160200191610214565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561013b5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b505050565b8060006000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106101b557805160ff19168380011785555b506101499291505b808211156101e557600081556001016101a1565b82800160010185558215610199579182015b828111156101995782518260005055916020019190600101906101c7565b5090565b820191906000526020600020905b8154815290600101906020018083116101f757829003601f168201915b50505050508156`, `[{"constant":false,"inputs":[{"name":"value","type":"string"}],"name":"SetField","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"field","outputs":[{"name":"","type":"string"}],"type":"function"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -478,6 +554,15 @@ var bindTests = []struct { } `, `6060604052346000575b6086806100176000396000f300606060405263ffffffff60e060020a60003504166349f8e98281146022575b6000565b34600057602c6055565b6040805173ffffffffffffffffffffffffffffffffffffffff9092168252519081900360200190f35b335b905600a165627a7a72305820aef6b7685c0fa24ba6027e4870404a57df701473fe4107741805c19f5138417c0029`, `[{"constant":true,"inputs":[],"name":"callFrom","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -538,6 +623,15 @@ var bindTests = []struct { } `, `6060604052341561000f57600080fd5b6103858061001e6000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806303a592131461009357806346546dbe146100c357806367e6633d146100ec5780639df4848514610181578063af7486ab146101b1578063b564b34d146101e1578063e02ab24d14610211578063e409ca4514610241575b600080fd5b341561009e57600080fd5b6100a6610271565b604051808381526020018281526020019250505060405180910390f35b34156100ce57600080fd5b6100d6610286565b6040518082815260200191505060405180910390f35b34156100f757600080fd5b6100ff61028e565b6040518083815260200180602001828103825283818151815260200191508051906020019080838360005b8381101561014557808201518184015260208101905061012a565b50505050905090810190601f1680156101725780820380516001836020036101000a031916815260200191505b50935050505060405180910390f35b341561018c57600080fd5b6101946102dc565b604051808381526020018281526020019250505060405180910390f35b34156101bc57600080fd5b6101c46102f1565b604051808381526020018281526020019250505060405180910390f35b34156101ec57600080fd5b6101f4610306565b604051808381526020018281526020019250505060405180910390f35b341561021c57600080fd5b61022461031b565b604051808381526020018281526020019250505060405180910390f35b341561024c57600080fd5b610254610330565b604051808381526020018281526020019250505060405180910390f35b60008060016002819150809050915091509091565b600080905090565b6000610298610345565b61013a8090506040805190810160405280600281526020017f7069000000000000000000000000000000000000000000000000000000000000815250915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b6020604051908101604052806000815250905600a165627a7a72305820d1a53d9de9d1e3d55cb3dc591900b63c4f1ded79114f7b79b332684840e186a40029`, `[{"constant":true,"inputs":[],"name":"LowerUpperCollision","outputs":[{"name":"_res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"_under_scored_func","outputs":[{"name":"_int","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UnderscoredOutput","outputs":[{"name":"_int","type":"int256"},{"name":"_string","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"PurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperLowerCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"AllPurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"__","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperUpperCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"LowerLowerCollision","outputs":[{"name":"_res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"}]`, + ` + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -608,6 +702,16 @@ var bindTests = []struct { `, `6060604052341561000f57600080fd5b61042c8061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063528300ff1461005c578063630c31e2146100fc578063c7d116dd14610156575b600080fd5b341561006757600080fd5b6100fa600480803590602001908201803590602001908080601f0160208091040260200160405190810160405280939291908181526020018383808284378201915050505050509190803590602001908201803590602001908080601f01602080910402602001604051908101604052809392919081815260200183838082843782019150505050505091905050610194565b005b341561010757600080fd5b610154600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091908035600019169060200190919080351515906020019091908035906020019091905050610367565b005b341561016157600080fd5b610192600480803590602001909190803560010b90602001909190803563ffffffff169060200190919050506103c3565b005b806040518082805190602001908083835b6020831015156101ca57805182526020820191506020810190506020830392506101a5565b6001836020036101000a0380198251168184511680821785525050505050509050019150506040518091039020826040518082805190602001908083835b60208310151561022d5780518252602082019150602081019050602083039250610208565b6001836020036101000a03801982511681845116808217855250505050505090500191505060405180910390207f3281fd4f5e152dd3385df49104a3f633706e21c9e80672e88d3bcddf33101f008484604051808060200180602001838103835285818151815260200191508051906020019080838360005b838110156102c15780820151818401526020810190506102a6565b50505050905090810190601f1680156102ee5780820380516001836020036101000a031916815260200191505b50838103825284818151815260200191508051906020019080838360005b8381101561032757808201518184015260208101905061030c565b50505050905090810190601f1680156103545780820380516001836020036101000a031916815260200191505b5094505050505060405180910390a35050565b81151583600019168573ffffffffffffffffffffffffffffffffffffffff167f1f097de4289df643bd9c11011cc61367aa12983405c021056e706eb5ba1250c8846040518082815260200191505060405180910390a450505050565b8063ffffffff168260010b847f3ca7f3a77e5e6e15e781850bc82e32adfa378a2a609370db24b4d0fae10da2c960405160405180910390a45050505600a165627a7a72305820d1f8a8bbddbc5bb29f285891d6ae1eef8420c52afdc05e1573f6114d8e1714710029`, `[{"constant":false,"inputs":[{"name":"str","type":"string"},{"name":"blob","type":"bytes"}],"name":"raiseDynamicEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"},{"name":"id","type":"bytes32"},{"name":"flag","type":"bool"},{"name":"value","type":"uint256"}],"name":"raiseSimpleEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"number","type":"uint256"},{"name":"short","type":"int16"},{"name":"long","type":"uint32"}],"name":"raiseNodataEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Addr","type":"address"},{"indexed":true,"name":"Id","type":"bytes32"},{"indexed":true,"name":"Flag","type":"bool"},{"indexed":false,"name":"Value","type":"uint256"}],"name":"SimpleEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Number","type":"uint256"},{"indexed":true,"name":"Short","type":"int16"},{"indexed":true,"name":"Long","type":"uint32"}],"name":"NodataEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"IndexedString","type":"string"},{"indexed":true,"name":"IndexedBytes","type":"bytes"},{"indexed":false,"name":"NonIndexedString","type":"string"},{"indexed":false,"name":"NonIndexedBytes","type":"bytes"}],"name":"DynamicEvent","type":"event"}]`, + ` + "math/big" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -757,6 +861,14 @@ var bindTests = []struct { `, `6060604052341561000f57600080fd5b6106438061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063344248551461005c5780638ed4573a1461011457806398ed1856146101ab575b600080fd5b341561006757600080fd5b610112600480806107800190600580602002604051908101604052809291906000905b828210156101055783826101800201600480602002604051908101604052809291906000905b828210156100f25783826060020160038060200260405190810160405280929190826003602002808284378201915050505050815260200190600101906100b0565b505050508152602001906001019061008a565b5050505091905050610208565b005b341561011f57600080fd5b61012761021d565b604051808260056000925b8184101561019b578284602002015160046000925b8184101561018d5782846020020151600360200280838360005b8381101561017c578082015181840152602081019050610161565b505050509050019260010192610147565b925050509260010192610132565b9250505091505060405180910390f35b34156101b657600080fd5b6101de6004808035906020019091908035906020019091908035906020019091905050610309565b604051808267ffffffffffffffff1667ffffffffffffffff16815260200191505060405180910390f35b80600090600561021992919061035f565b5050565b6102256103b0565b6000600580602002604051908101604052809291906000905b8282101561030057838260040201600480602002604051908101604052809291906000905b828210156102ed578382016003806020026040519081016040528092919082600380156102d9576020028201916000905b82829054906101000a900467ffffffffffffffff1667ffffffffffffffff16815260200190600801906020826007010492830192600103820291508084116102945790505b505050505081526020019060010190610263565b505050508152602001906001019061023e565b50505050905090565b60008360058110151561031857fe5b600402018260048110151561032957fe5b018160038110151561033757fe5b6004918282040191900660080292509250509054906101000a900467ffffffffffffffff1681565b826005600402810192821561039f579160200282015b8281111561039e5782518290600461038e9291906103df565b5091602001919060040190610375565b5b5090506103ac919061042d565b5090565b610780604051908101604052806005905b6103c9610459565b8152602001906001900390816103c15790505090565b826004810192821561041c579160200282015b8281111561041b5782518290600361040b929190610488565b50916020019190600101906103f2565b5b5090506104299190610536565b5090565b61045691905b8082111561045257600081816104499190610562565b50600401610433565b5090565b90565b610180604051908101604052806004905b6104726105a7565b81526020019060019003908161046a5790505090565b82600380016004900481019282156105255791602002820160005b838211156104ef57835183826101000a81548167ffffffffffffffff021916908367ffffffffffffffff16021790555092602001926008016020816007010492830192600103026104a3565b80156105235782816101000a81549067ffffffffffffffff02191690556008016020816007010492830192600103026104ef565b505b50905061053291906105d9565b5090565b61055f91905b8082111561055b57600081816105529190610610565b5060010161053c565b5090565b90565b50600081816105719190610610565b50600101600081816105839190610610565b50600101600081816105959190610610565b5060010160006105a59190610610565b565b6060604051908101604052806003905b600067ffffffffffffffff168152602001906001900390816105b75790505090565b61060d91905b8082111561060957600081816101000a81549067ffffffffffffffff0219169055506001016105df565b5090565b90565b50600090555600a165627a7a7230582087e5a43f6965ab6ef7a4ff056ab80ed78fd8c15cff57715a1bf34ec76a93661c0029`, `[{"constant":false,"inputs":[{"name":"arr","type":"uint64[3][4][5]"}],"name":"storeDeepUintArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"retrieveDeepArray","outputs":[{"name":"","type":"uint64[3][4][5]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"}],"name":"deepUint64Array","outputs":[{"name":"","type":"uint64"}],"payable":false,"stateMutability":"view","type":"function"}]`, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, ` // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() @@ -819,15 +931,6 @@ func TestBindings(t *testing.T) { if !common.FileExist(gocmd) { t.Skip("go sdk not found for testing") } - // Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845) - linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil,uint64(10000000000)))\n}") - linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil) - if err != nil { - t.Fatalf("failed check for goimports symlink bug: %v", err) - } - if !strings.Contains(string(linkTestDeps), "go-ethereum") { - t.Skip("symlinked environment doesn't support bind (https://github.com/golang/go/issues/14845)") - } // Create a temporary workspace for the test suite ws, err := ioutil.TempDir("", "") if err != nil { @@ -850,12 +953,19 @@ func TestBindings(t *testing.T) { t.Fatalf("test %d: failed to write binding: %v", i, err) } // Generate the test file with the injected test code - code := fmt.Sprintf("package bindtest\nimport \"testing\"\nfunc Test%s(t *testing.T){\n%s\n}", tt.name, tt.tester) - blob, err := imports.Process("", []byte(code), nil) - if err != nil { - t.Fatalf("test %d: failed to generate tests: %v", i, err) - } - if err := ioutil.WriteFile(filepath.Join(pkg, strings.ToLower(tt.name)+"_test.go"), blob, 0600); err != nil { + code := fmt.Sprintf(` + package bindtest + + import ( + "testing" + %s + ) + + func Test%s(t *testing.T) { + %s + } + `, tt.imports, tt.name, tt.tester) + if err := ioutil.WriteFile(filepath.Join(pkg, strings.ToLower(tt.name)+"_test.go"), []byte(code), 0600); err != nil { t.Fatalf("test %d: failed to write tests: %v", i, err) } } diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index 7202ee67a1..02d0258e0c 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -64,6 +64,30 @@ const tmplSourceGo = ` package {{.Package}} +import ( + "math/big" + "strings" + + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" +) + +// Reference imports to suppress errors if they are not otherwise used. +var ( + _ = big.NewInt + _ = strings.NewReader + _ = ethereum.NotFound + _ = abi.U256 + _ = bind.Bind + _ = common.Big1 + _ = types.BloomLookup + _ = event.NewSubscription +) + {{range $contract := .Contracts}} // {{.Type}}ABI is the input ABI used to generate the binding from. const {{.Type}}ABI = "{{.InputABI}}" diff --git a/accounts/hd.go b/accounts/hd.go index 277f688e48..6ed6318078 100644 --- a/accounts/hd.go +++ b/accounts/hd.go @@ -30,8 +30,8 @@ import ( var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0} // DefaultBaseDerivationPath is the base path from which custom derivation endpoints -// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second -// at m/44'/60'/0'/1, etc. +// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second +// at m/44'/60'/0'/0/1, etc. var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0} // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints diff --git a/accounts/keystore/key.go b/accounts/keystore/key.go index 9e3e4856c4..0564751c43 100644 --- a/accounts/keystore/key.go +++ b/accounts/keystore/key.go @@ -66,19 +66,19 @@ type plainKeyJSON struct { type encryptedKeyJSONV3 struct { Address string `json:"address"` - Crypto cryptoJSON `json:"crypto"` + Crypto CryptoJSON `json:"crypto"` Id string `json:"id"` Version int `json:"version"` } type encryptedKeyJSONV1 struct { Address string `json:"address"` - Crypto cryptoJSON `json:"crypto"` + Crypto CryptoJSON `json:"crypto"` Id string `json:"id"` Version string `json:"version"` } -type cryptoJSON struct { +type CryptoJSON struct { Cipher string `json:"cipher"` CipherText string `json:"ciphertext"` CipherParams cipherparamsJSON `json:"cipherparams"` diff --git a/accounts/keystore/keystore_passphrase.go b/accounts/keystore/keystore_passphrase.go index 5aa3a6bbd8..9794f32fe1 100644 --- a/accounts/keystore/keystore_passphrase.go +++ b/accounts/keystore/keystore_passphrase.go @@ -135,29 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string { return filepath.Join(ks.keysDirPath, filename) } -// EncryptKey encrypts a key using the specified scrypt parameters into a json -// blob that can be decrypted later on. -func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) { - authArray := []byte(auth) +// Encryptdata encrypts the data given as 'data' with the password 'auth'. +func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) { salt := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, salt); err != nil { panic("reading from crypto/rand failed: " + err.Error()) } - derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen) + derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen) if err != nil { - return nil, err + return CryptoJSON{}, err } encryptKey := derivedKey[:16] - keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32) iv := make([]byte, aes.BlockSize) // 16 if _, err := io.ReadFull(rand.Reader, iv); err != nil { panic("reading from crypto/rand failed: " + err.Error()) } - cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv) + cipherText, err := aesCTRXOR(encryptKey, data, iv) if err != nil { - return nil, err + return CryptoJSON{}, err } mac := crypto.Keccak256(derivedKey[16:32], cipherText) @@ -167,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) { scryptParamsJSON["p"] = scryptP scryptParamsJSON["dklen"] = scryptDKLen scryptParamsJSON["salt"] = hex.EncodeToString(salt) - cipherParamsJSON := cipherparamsJSON{ IV: hex.EncodeToString(iv), } - cryptoStruct := cryptoJSON{ + cryptoStruct := CryptoJSON{ Cipher: "aes-128-ctr", CipherText: hex.EncodeToString(cipherText), CipherParams: cipherParamsJSON, @@ -180,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) { KDFParams: scryptParamsJSON, MAC: hex.EncodeToString(mac), } + return cryptoStruct, nil +} + +// EncryptKey encrypts a key using the specified scrypt parameters into a json +// blob that can be decrypted later on. +func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) { + keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32) + cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP) + if err != nil { + return nil, err + } encryptedKeyJSONV3 := encryptedKeyJSONV3{ hex.EncodeToString(key.Address[:]), cryptoStruct, @@ -226,43 +233,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) { PrivateKey: key, }, nil } +func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) { + if cryptoJson.Cipher != "aes-128-ctr" { + return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher) + } + mac, err := hex.DecodeString(cryptoJson.MAC) + if err != nil { + return nil, err + } + + iv, err := hex.DecodeString(cryptoJson.CipherParams.IV) + if err != nil { + return nil, err + } + + cipherText, err := hex.DecodeString(cryptoJson.CipherText) + if err != nil { + return nil, err + } + + derivedKey, err := getKDFKey(cryptoJson, auth) + if err != nil { + return nil, err + } + + calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText) + if !bytes.Equal(calculatedMAC, mac) { + return nil, ErrDecrypt + } + + plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv) + if err != nil { + return nil, err + } + return plainText, err +} func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) { if keyProtected.Version != version { return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version) } - - if keyProtected.Crypto.Cipher != "aes-128-ctr" { - return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher) - } - keyId = uuid.Parse(keyProtected.Id) - mac, err := hex.DecodeString(keyProtected.Crypto.MAC) - if err != nil { - return nil, nil, err - } - - iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV) - if err != nil { - return nil, nil, err - } - - cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText) - if err != nil { - return nil, nil, err - } - - derivedKey, err := getKDFKey(keyProtected.Crypto, auth) - if err != nil { - return nil, nil, err - } - - calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText) - if !bytes.Equal(calculatedMAC, mac) { - return nil, nil, ErrDecrypt - } - - plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv) + plainText, err := DecryptDataV3(keyProtected.Crypto, auth) if err != nil { return nil, nil, err } @@ -303,7 +315,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt return plainText, keyId, err } -func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) { +func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) { authArray := []byte(auth) salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string)) if err != nil { diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 2583fbc4da..7d5f67908d 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction signer = new(types.HomesteadSigner) } else { signer = types.NewEIP155Signer(chainID) - signature[64] = signature[64] - byte(chainID.Uint64()*2+35) + signature[64] -= byte(chainID.Uint64()*2 + 35) } signed, err := tx.WithSignature(signer, signature) if err != nil { diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index b84a955992..a9d2e9959e 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction signer = new(types.HomesteadSigner) } else { signer = types.NewEIP155Signer(chainID) - signature[64] = signature[64] - byte(chainID.Uint64()*2+35) + signature[64] -= byte(chainID.Uint64()*2 + 35) } // Inject the final signature into the transaction and sanity check the sender signed, err := tx.WithSignature(signer, signature) diff --git a/appveyor.yml b/appveyor.yml index b056cb3fd7..e5126b2529 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.windows-%GETH_ARCH%.zip - - 7z x go1.11.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip + - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/build/ci.go b/build/ci.go index 40252cbde1..1bbc944714 100644 --- a/build/ci.go +++ b/build/ci.go @@ -320,9 +320,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd // "tests" also includes static analysis tools such as vet. func doTest(cmdline []string) { - var ( - coverage = flag.Bool("coverage", false, "Whether to record code coverage") - ) + coverage := flag.Bool("coverage", false, "Whether to record code coverage") flag.CommandLine.Parse(cmdline) env := build.Env() @@ -332,14 +330,11 @@ func doTest(cmdline []string) { } packages = build.ExpandPackagesNoVendor(packages) - // Run analysis tools before the tests. - build.MustRun(goTool("vet", packages...)) - // Run the actual tests. - gotest := goTool("test", buildFlags(env)...) // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. - gotest.Args = append(gotest.Args, "-p", "1") + gotest := goTool("test", buildFlags(env)...) + gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m") if *coverage { gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") } @@ -1040,7 +1035,7 @@ func xgoTool(args []string) *exec.Cmd { func doPurge(cmdline []string) { var ( store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) - limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`) + limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) ) flag.CommandLine.Parse(cmdline) diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 3ac37ba7ad..d19164b18d 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -75,7 +75,7 @@ func main() { bins []string types []string ) - if *solFlag != "" || *abiFlag == "-" { + if *solFlag != "" || (*abiFlag == "-" && *pkgFlag == "") { // Generate the list of types to exclude from binding exclude := make(map[string]bool) for _, kind := range strings.Split(*excFlag, ",") { @@ -111,7 +111,13 @@ func main() { } } else { // Otherwise load up the ABI, optional bytecode and type name from the parameters - abi, err := ioutil.ReadFile(*abiFlag) + var abi []byte + var err error + if *abiFlag == "-" { + abi, err = ioutil.ReadAll(os.Stdin) + } else { + abi, err = ioutil.ReadFile(*abiFlag) + } if err != nil { fmt.Printf("Failed to read input ABI: %v\n", err) os.Exit(-1) @@ -155,6 +161,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) { if err != nil { return nil, err } - return compiler.ParseCombinedJSON(bytes, "", "", "", "") } diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 8459008652..32f7d63bea 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -38,7 +38,7 @@ func main() { var ( listenAddr = flag.String("addr", ":30301", "listen address") genKey = flag.String("genkey", "", "generate a node key") - writeAddr = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit") + writeAddr = flag.Bool("writeaddress", false, "write out the node's public key and quit") nodeKeyFile = flag.String("nodekey", "", "private key filename") nodeKeyHex = flag.String("nodekeyhex", "", "private key as hex (for testing)") natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:)") @@ -86,7 +86,7 @@ func main() { } if *writeAddr { - fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey)) + fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:]) os.Exit(0) } @@ -119,16 +119,17 @@ func main() { } if *runv5 { - if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil { + if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil { utils.Fatalf("%v", err) } } else { + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, nodeKey) cfg := discover.Config{ - PrivateKey: nodeKey, - AnnounceAddr: realaddr, - NetRestrict: restrictList, + PrivateKey: nodeKey, + NetRestrict: restrictList, } - if _, err := discover.ListenUDP(conn, cfg); err != nil { + if _, err := discover.ListenUDP(conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } diff --git a/cmd/clef/README.md b/cmd/clef/README.md index 027c22c98f..c9461be101 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -91,7 +91,7 @@ invoking methods with the following info: * [x] Version info about the signer * [x] Address of API (http/ipc) * [ ] List of known accounts -* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected. +* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected. * [ ] `account_signRawTransaction` * [ ] `account_bulkSignTransactions([] transactions)` should * only exist if enabled via config/flag @@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification). Some of these call can require user interaction. Clients must be aware that responses -may be delayed significanlty or may never be received if a users decides to ignore the confirmation request. +may be delayed significantly or may never be received if a users decides to ignore the confirmation request. The External API is **untrusted** : it does not accept credentials over this api, nor does it expect that requests have any authority. @@ -862,7 +862,7 @@ A UI should conform to the following rules. * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts * The signer provides accounts -* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled +* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled along with the UI. @@ -875,3 +875,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | +| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)| diff --git a/cmd/clef/docs/qubes/clef_qubes_http.png b/cmd/clef/docs/qubes/clef_qubes_http.png index a641e1987f..e95ad8da4a 100644 Binary files a/cmd/clef/docs/qubes/clef_qubes_http.png and b/cmd/clef/docs/qubes/clef_qubes_http.png differ diff --git a/cmd/clef/docs/qubes/clef_qubes_qrexec.png b/cmd/clef/docs/qubes/clef_qubes_qrexec.png index f57fc8933e..b1814e7c36 100644 Binary files a/cmd/clef/docs/qubes/clef_qubes_qrexec.png and b/cmd/clef/docs/qubes/clef_qubes_qrexec.png differ diff --git a/cmd/clef/docs/qubes/qrexec-example.png b/cmd/clef/docs/qubes/qrexec-example.png index 0d86fde19d..fc5d57725d 100644 Binary files a/cmd/clef/docs/qubes/qrexec-example.png and b/cmd/clef/docs/qubes/qrexec-example.png differ diff --git a/cmd/clef/docs/qubes/qubes_newaccount-1.png b/cmd/clef/docs/qubes/qubes_newaccount-1.png index 598dbbee7a..3bfc8b5b7e 100644 Binary files a/cmd/clef/docs/qubes/qubes_newaccount-1.png and b/cmd/clef/docs/qubes/qubes_newaccount-1.png differ diff --git a/cmd/clef/docs/qubes/qubes_newaccount-2.png b/cmd/clef/docs/qubes/qubes_newaccount-2.png index cd762a1934..c6dbd535dd 100644 Binary files a/cmd/clef/docs/qubes/qubes_newaccount-2.png and b/cmd/clef/docs/qubes/qubes_newaccount-2.png differ diff --git a/cmd/clef/intapi_changelog.md b/cmd/clef/intapi_changelog.md index 7d2a897ea2..92a39a268f 100644 --- a/cmd/clef/intapi_changelog.md +++ b/cmd/clef/intapi_changelog.md @@ -1,5 +1,24 @@ ### Changelog for internal API (ui-api) +### 3.0.0 + +* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup + +### 2.1.0 + +* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords. + +The following structures are used: +```golang + UserInputRequest struct { + Prompt string `json:"prompt"` + Title string `json:"title"` + IsPassword bool `json:"isPassword"` + } + UserInputResponse struct { + Text string `json:"text"` + } + ### 2.0.0 * Modify how `call_info` on a transaction is conveyed. New format: diff --git a/cmd/clef/main.go b/cmd/clef/main.go index c060285be6..519d63b3c1 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -35,8 +35,10 @@ import ( "runtime" "strings" + "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/console" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -48,10 +50,10 @@ import ( ) // ExternalAPIVersion -- see extapi_changelog.md -const ExternalAPIVersion = "3.0.0" +const ExternalAPIVersion = "4.0.0" // InternalAPIVersion -- see intapi_changelog.md -const InternalAPIVersion = "2.0.0" +const InternalAPIVersion = "3.0.0" const legalWarning = ` WARNING! @@ -91,7 +93,7 @@ var ( } signerSecretFlag = cli.StringFlag{ Name: "signersecret", - Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash", + Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash", } dBFlag = cli.StringFlag{ Name: "4bytedb", @@ -155,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell Clef that the file is 'safe' to execute.`, } - addCredentialCommand = cli.Command{ - Action: utils.MigrateFlags(addCredential), - Name: "addpw", + setCredentialCommand = cli.Command{ + Action: utils.MigrateFlags(setCredential), + Name: "setpw", Usage: "Store a credential for a keystore file", - ArgsUsage: "
", + ArgsUsage: "
", Flags: []cli.Flag{ logLevelFlag, configdirFlag, signerSecretFlag, }, Description: ` -The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will + The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will remove any stored credential for that address (keyfile) `, } @@ -198,7 +200,7 @@ func init() { advancedMode, } app.Action = signer - app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand} + app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand} } func main() { @@ -212,25 +214,45 @@ func initializeSecrets(c *cli.Context) error { if err := initialize(c); err != nil { return err } - configDir := c.String(configdirFlag.Name) + configDir := c.GlobalString(configdirFlag.Name) masterSeed := make([]byte, 256) - n, err := io.ReadFull(rand.Reader, masterSeed) + num, err := io.ReadFull(rand.Reader, masterSeed) if err != nil { return err } - if n != len(masterSeed) { + if num != len(masterSeed) { return fmt.Errorf("failed to read enough random") } + + n, p := keystore.StandardScryptN, keystore.StandardScryptP + if c.GlobalBool(utils.LightKDFFlag.Name) { + n, p = keystore.LightScryptN, keystore.LightScryptP + } + text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password." + var password string + for { + password = getPassPhrase(text, true) + if err := core.ValidatePasswordFormat(password); err != nil { + fmt.Printf("invalid password: %v\n", err) + } else { + break + } + } + cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p) + if err != nil { + return fmt.Errorf("failed to encrypt master seed: %v", err) + } + err = os.Mkdir(configDir, 0700) if err != nil && !os.IsExist(err) { return err } - location := filepath.Join(configDir, "secrets.dat") + location := filepath.Join(configDir, "masterseed.json") if _, err := os.Stat(location); err == nil { return fmt.Errorf("file %v already exists, will not overwrite", location) } - err = ioutil.WriteFile(location, masterSeed, 0400) + err = ioutil.WriteFile(location, cipherSeed, 0400) if err != nil { return err } @@ -255,11 +277,11 @@ func attestFile(ctx *cli.Context) error { return err } - stretchedKey, err := readMasterKey(ctx) + stretchedKey, err := readMasterKey(ctx, nil) if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.String(configdirFlag.Name) + configDir := ctx.GlobalString(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) confKey := crypto.Keccak256([]byte("config"), stretchedKey) @@ -271,38 +293,36 @@ func attestFile(ctx *cli.Context) error { return nil } -func addCredential(ctx *cli.Context) error { +func setCredential(ctx *cli.Context) error { if len(ctx.Args()) < 1 { - utils.Fatalf("This command requires at leaste one argument.") + utils.Fatalf("This command requires an address to be passed as an argument.") } if err := initialize(ctx); err != nil { return err } - stretchedKey, err := readMasterKey(ctx) + address := ctx.Args().First() + password := getPassPhrase("Enter a passphrase to store with this address.", true) + + stretchedKey, err := readMasterKey(ctx, nil) if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.String(configdirFlag.Name) + configDir := ctx.GlobalString(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) // Initialize the encrypted storages pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) - key := ctx.Args().First() - value := "" - if len(ctx.Args()) > 1 { - value = ctx.Args().Get(1) - } - pwStorage.Put(key, value) - log.Info("Credential store updated", "key", key) + pwStorage.Put(address, password) + log.Info("Credential store updated", "key", address) return nil } func initialize(c *cli.Context) error { // Set up the logger to print everything logOutput := os.Stdout - if c.Bool(stdiouiFlag.Name) { + if c.GlobalBool(stdiouiFlag.Name) { logOutput = os.Stderr // If using the stdioui, we can't do the 'confirm'-flow fmt.Fprintf(logOutput, legalWarning) @@ -323,26 +343,28 @@ func signer(c *cli.Context) error { var ( ui core.SignerUI ) - if c.Bool(stdiouiFlag.Name) { + if c.GlobalBool(stdiouiFlag.Name) { log.Info("Using stdin/stdout as UI-channel") ui = core.NewStdIOUI() } else { log.Info("Using CLI as UI-channel") ui = core.NewCommandlineUI() } - db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name)) + fourByteDb := c.GlobalString(dBFlag.Name) + fourByteLocal := c.GlobalString(customDBFlag.Name) + db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal) if err != nil { utils.Fatalf(err.Error()) } - log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb")) + log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal) var ( api core.ExternalAPI ) - configDir := c.String(configdirFlag.Name) - if stretchedKey, err := readMasterKey(c); err != nil { - log.Info("No master seed provided, rules disabled") + configDir := c.GlobalString(configdirFlag.Name) + if stretchedKey, err := readMasterKey(c, ui); err != nil { + log.Info("No master seed provided, rules disabled", "error", err) } else { if err != nil { @@ -361,7 +383,7 @@ func signer(c *cli.Context) error { configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) //Do we have a rule-file? - ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name)) + ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name)) if err != nil { log.Info("Could not load rulefile, rules not enabled", "file", "rulefile") } else { @@ -385,17 +407,15 @@ func signer(c *cli.Context) error { } apiImpl := core.NewSignerAPI( - c.Int64(utils.NetworkIdFlag.Name), - c.String(keystoreFlag.Name), - c.Bool(utils.NoUSBFlag.Name), + c.GlobalInt64(utils.NetworkIdFlag.Name), + c.GlobalString(keystoreFlag.Name), + c.GlobalBool(utils.NoUSBFlag.Name), ui, db, - c.Bool(utils.LightKDFFlag.Name), - c.Bool(advancedMode.Name)) - + c.GlobalBool(utils.LightKDFFlag.Name), + c.GlobalBool(advancedMode.Name)) api = apiImpl - // Audit logging - if logfile := c.String(auditLogFlag.Name); logfile != "" { + if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" { api, err = core.NewAuditLogger(logfile, api) if err != nil { utils.Fatalf(err.Error()) @@ -414,13 +434,13 @@ func signer(c *cli.Context) error { Service: api, Version: "1.0"}, } - if c.Bool(utils.RPCEnabledFlag.Name) { + if c.GlobalBool(utils.RPCEnabledFlag.Name) { vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name)) cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name)) // start http server - httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name)) + httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name)) listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) @@ -434,9 +454,9 @@ func signer(c *cli.Context) error { }() } - if !c.Bool(utils.IPCDisabledFlag.Name) { + if !c.GlobalBool(utils.IPCDisabledFlag.Name) { if c.IsSet(utils.IPCPathFlag.Name) { - ipcapiURL = c.String(utils.IPCPathFlag.Name) + ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name) } else { ipcapiURL = filepath.Join(configDir, "clef.ipc") } @@ -453,7 +473,7 @@ func signer(c *cli.Context) error { } - if c.Bool(testFlag.Name) { + if c.GlobalBool(testFlag.Name) { log.Info("Performing UI test") go testExternalUI(apiImpl) } @@ -512,36 +532,52 @@ func homeDir() string { } return "" } -func readMasterKey(ctx *cli.Context) ([]byte, error) { +func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) { var ( file string - configDir = ctx.String(configdirFlag.Name) + configDir = ctx.GlobalString(configdirFlag.Name) ) - if ctx.IsSet(signerSecretFlag.Name) { - file = ctx.String(signerSecretFlag.Name) + if ctx.GlobalIsSet(signerSecretFlag.Name) { + file = ctx.GlobalString(signerSecretFlag.Name) } else { - file = filepath.Join(configDir, "secrets.dat") + file = filepath.Join(configDir, "masterseed.json") } if err := checkFile(file); err != nil { return nil, err } - masterKey, err := ioutil.ReadFile(file) + cipherKey, err := ioutil.ReadFile(file) if err != nil { return nil, err } - if len(masterKey) < 256 { - return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey)) + var password string + // If ui is not nil, get the password from ui. + if ui != nil { + resp, err := ui.OnInputRequired(core.UserInputRequest{ + Title: "Master Password", + Prompt: "Please enter the password to decrypt the master seed", + IsPassword: true}) + if err != nil { + return nil, err + } + password = resp.Text + } else { + password = getPassPhrase("Decrypt master seed of clef", false) } + masterSeed, err := decryptSeed(cipherKey, password) + if err != nil { + return nil, fmt.Errorf("failed to decrypt the master seed of clef") + } + if len(masterSeed) < 256 { + return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed)) + } + // Create vault location - vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10])) + vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10])) err = os.Mkdir(vaultLocation, 0700) if err != nil && !os.IsExist(err) { return nil, err } - //!TODO, use KDF to stretch the master key - // stretched_key := stretch_key(master_key) - - return masterKey, nil + return masterSeed, nil } // checkFile is a convenience function to check if a file @@ -619,6 +655,59 @@ func testExternalUI(api *core.SignerAPI) { } +// getPassPhrase retrieves the password associated with clef, either fetched +// from a list of preloaded passphrases, or requested interactively from the user. +// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one. +func getPassPhrase(prompt string, confirmation bool) string { + fmt.Println(prompt) + password, err := console.Stdin.PromptPassword("Passphrase: ") + if err != nil { + utils.Fatalf("Failed to read passphrase: %v", err) + } + if confirmation { + confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ") + if err != nil { + utils.Fatalf("Failed to read passphrase confirmation: %v", err) + } + if password != confirm { + utils.Fatalf("Passphrases do not match") + } + } + return password +} + +type encryptedSeedStorage struct { + Description string `json:"description"` + Version int `json:"version"` + Params keystore.CryptoJSON `json:"params"` +} + +// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping, +// to encrypt the master seed +func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) { + cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP) + if err != nil { + return nil, err + } + return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct}) +} + +// decryptSeed decrypts the master seed +func decryptSeed(keyjson []byte, auth string) ([]byte, error) { + var encSeed encryptedSeedStorage + if err := json.Unmarshal(keyjson, &encSeed); err != nil { + return nil, err + } + if encSeed.Version != 1 { + log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version)) + } + seed, err := keystore.DecryptDataV3(encSeed.Params, auth) + if err != nil { + return nil, err + } + return seed, err +} + /** //Create Account diff --git a/cmd/clef/sign_flow.png b/cmd/clef/sign_flow.png index 9c0f3cc5d5..93ef81a32e 100644 Binary files a/cmd/clef/sign_flow.png and b/cmd/clef/sign_flow.png differ diff --git a/cmd/clef/tutorial.md b/cmd/clef/tutorial.md index ed86f4810e..dfb31ba4ef 100644 --- a/cmd/clef/tutorial.md +++ b/cmd/clef/tutorial.md @@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ ## Creating rules -Now, you can create a rule-file. +Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy. ```javascript function ApproveListing(){ return "Approve" } ``` -Get the `sha256` hash.... + +Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`... ```text #sha256sum rules.js 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 rules.js ``` -...And then `attest` the file: +...now `attest` the file... ```text #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 INFO [02-21|12:14:38] Ruleset attestation updated sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 ``` -At this point, we then start the signer with the rule-file: +...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory: ```text -#./signer --rules rules.js +#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD +``` -INFO [02-21|12:15:18] Using CLI as UI-channel -INFO [02-21|12:15:18] Loaded 4byte db signatures=5509 file=./4byte.json -INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile -DEBUG[02-21|12:15:18] FS scan times list=35.335µs set=5.536µs diff=5.073µs -DEBUG[02-21|12:15:18] Ledger support enabled -DEBUG[02-21|12:15:18] Trezor support enabled -INFO [02-21|12:15:18] Audit logs configured file=audit.log -INFO [02-21|12:15:18] HTTP endpoint opened url=http://localhost:8550 +At this point, we can start the signer with the rule-file: +```text +#./signer --rules rules.js --rpc + +INFO [09-25|20:28:11.866] Using CLI as UI-channel +INFO [09-25|20:28:11.876] Loaded 4byte db signatures=5509 file=./4byte.json +INFO [09-25|20:28:11.877] Rule engine configured file=./rules.js +DEBUG[09-25|20:28:11.877] FS scan times list=100.781µs set=13.253µs diff=5.761µs +DEBUG[09-25|20:28:11.884] Ledger support enabled +DEBUG[09-25|20:28:11.888] Trezor support enabled +INFO [09-25|20:28:11.888] Audit logs configured file=audit.log +DEBUG[09-25|20:28:11.888] HTTP registered namespace=account +INFO [09-25|20:28:11.890] HTTP endpoint opened url=http://localhost:8550 +DEBUG[09-25|20:28:11.890] IPC registered namespace=account +INFO [09-25|20:28:11.890] IPC endpoint opened url= ------- Signer info ------- +* extapi_version : 2.0.0 +* intapi_version : 2.0.0 * extapi_http : http://localhost:8550 * extapi_ipc : -* extapi_version : 2.0.0 -* intapi_version : 1.2.0 - ``` Any list-requests will now be auto-approved by our rule-file. @@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things: ## Adding credentials -In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores. +In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores. ```text -#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test +#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password" INFO [02-21|13:43:21] Credential store updated key=0x694267f14675d7e1b9494fd8d72fefe1755710fa ``` ## More advanced rules -Now let's update the rules to make use of credentials +Now let's update the rules to make use of credentials: ```javascript function ApproveListing(){ @@ -134,13 +142,15 @@ function ApproveSignData(r){ } ``` -In this example, -* any requests to sign data with the account `0x694...` will be - * auto-approved if the message contains with `bazonk`, - * and auto-rejected if it does not. - * Any other signing-requests will be passed along for manual approve/reject. +In this example: +* Any requests to sign data with the account `0x694...` will be + * auto-approved if the message contains with `bazonk` + * auto-rejected if it does not. +* Any other signing-requests will be passed along for manual approve/reject. -..attest the new file +_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._ + +Attest the new file... ```text #sha256sum rules.js 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f rules.js @@ -155,21 +165,24 @@ And start the signer: ``` #./signer --rules rules.js --rpc -INFO [02-21|14:41:56] Using CLI as UI-channel -INFO [02-21|14:41:56] Loaded 4byte db signatures=5509 file=./4byte.json -INFO [02-21|14:41:56] Rule engine configured file=rules.js -DEBUG[02-21|14:41:56] FS scan times list=34.607µs set=4.509µs diff=4.87µs -DEBUG[02-21|14:41:56] Ledger support enabled -DEBUG[02-21|14:41:56] Trezor support enabled -INFO [02-21|14:41:56] Audit logs configured file=audit.log -INFO [02-21|14:41:56] HTTP endpoint opened url=http://localhost:8550 +INFO [09-25|21:02:16.450] Using CLI as UI-channel +INFO [09-25|21:02:16.466] Loaded 4byte db signatures=5509 file=./4byte.json +INFO [09-25|21:02:16.467] Rule engine configured file=./rules.js +DEBUG[09-25|21:02:16.468] FS scan times list=1.45262ms set=21.926µs diff=6.944µs +DEBUG[09-25|21:02:16.473] Ledger support enabled +DEBUG[09-25|21:02:16.475] Trezor support enabled +INFO [09-25|21:02:16.476] Audit logs configured file=audit.log +DEBUG[09-25|21:02:16.476] HTTP registered namespace=account +INFO [09-25|21:02:16.478] HTTP endpoint opened url=http://localhost:8550 +DEBUG[09-25|21:02:16.478] IPC registered namespace=account +INFO [09-25|21:02:16.478] IPC endpoint opened url= ------- Signer info ------- * extapi_version : 2.0.0 -* intapi_version : 1.2.0 +* intapi_version : 2.0.0 * extapi_http : http://localhost:8550 * extapi_ipc : -INFO [02-21|14:41:56] error occurred during execution error="ReferenceError: 'OnSignerStartup' is not defined" ``` + And then test signing, once with `bazonk` and once without: ``` @@ -195,4 +208,4 @@ t=2018-02-21T14:42:41+0100 lvl=info msg=Sign api=signer type=request meta t=2018-02-21T14:42:42+0100 lvl=info msg=Sign api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil t=2018-02-21T14:42:56+0100 lvl=info msg=Sign api=signer type=request metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a t=2018-02-21T14:42:56+0100 lvl=info msg=Sign api=signer type=response data= error="Request denied" -``` +``` \ No newline at end of file diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go index f16424fbe6..50cb4f0e42 100644 --- a/cmd/evm/json_logger.go +++ b/cmd/evm/json_logger.go @@ -45,14 +45,15 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create // CaptureState outputs state information on the logger. func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { log := vm.StructLog{ - Pc: pc, - Op: op, - Gas: gas, - GasCost: cost, - MemorySize: memory.Len(), - Storage: nil, - Depth: depth, - Err: err, + Pc: pc, + Op: op, + Gas: gas, + GasCost: cost, + MemorySize: memory.Len(), + Storage: nil, + Depth: depth, + RefundCounter: env.StateDB.GetRefund(), + Err: err, } if !l.cfg.DisableMemory { log.Memory = memory.Data() diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6d5ff069f6..06c9be380a 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} state, err := test.Run(st, cfg) + // print state root for evmlab tracing + if ctx.GlobalBool(MachineFlag.Name) && state != nil { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + } if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() @@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error { result.State = &dump } } - // print state root for evmlab tracing (already committed above, so no need to delete objects again - if ctx.GlobalBool(MachineFlag.Name) && state != nil { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) - } results = append(results, *result) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ea28d9a69c..7e6004eea2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -21,7 +21,6 @@ import ( "fmt" "math" "os" - "runtime" godebug "runtime/debug" "sort" "strconv" @@ -93,6 +92,7 @@ var ( utils.LightKDFFlag, utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, utils.TrieCacheGenFlag, utils.ListenPortFlag, @@ -212,8 +212,6 @@ func init() { app.Flags = append(app.Flags, metricsFlags...) app.Before = func(ctx *cli.Context) error { - runtime.GOMAXPROCS(runtime.NumCPU()) - logdir := "" if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) { logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs") diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index eb5faf6582..de212e012b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -133,6 +133,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, utils.TrieCacheGenFlag, }, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 038152a3e4..069adfe4f4 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -99,6 +99,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{ "NetworkID": config.network, "Port": config.port, + "IP": client.address, "Peers": config.peersTotal, "LightFlag": lightFlag, "Bootnodes": strings.Join(bootnodes, ","), @@ -227,10 +228,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) // Container available, retrieve its node ID and its genesis json var out []byte - if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil { + if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil { return nil, ErrServiceUnreachable } - id := bytes.Trim(bytes.TrimSpace(out), "\"") + enode := bytes.Trim(bytes.TrimSpace(out), "\"") if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil { return nil, ErrServiceUnreachable @@ -265,7 +266,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) gasLimit: gasLimit, gasPrice: gasPrice, } - stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port) + stats.enode = string(enode) return stats, nil } diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index 67e852dde5..072541b659 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -29,7 +29,65 @@ import ( "gopkg.in/urfave/cli.v1" ) -var salt = make([]byte, 32) +var ( + salt = make([]byte, 32) + accessCommand = cli.Command{ + CustomHelpTemplate: helpTemplate, + Name: "access", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root manifest", + Subcommands: []cli.Command{ + { + CustomHelpTemplate: helpTemplate, + Name: "new", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + Subcommands: []cli.Command{ + { + Action: accessNewPass, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + }, + Name: "pass", + Usage: "encrypts a reference with a password and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewPK, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + SwarmAccessGrantKeyFlag, + }, + Name: "pk", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewACT, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + SwarmAccessGrantKeysFlag, + SwarmDryRunFlag, + utils.PasswordFileFlag, + }, + Name: "act", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + }, + }, + }, + } +) func init() { if _, err := io.ReadFull(rand.Reader, salt); err != nil { @@ -56,6 +114,9 @@ func accessNewPass(ctx *cli.Context) { utils.Fatalf("error getting session key: %v", err) } m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if err != nil { + utils.Fatalf("had an error generating the manifest: %v", err) + } if dryRun { err = printManifests(m, nil) if err != nil { @@ -89,6 +150,9 @@ func accessNewPK(ctx *cli.Context) { utils.Fatalf("error getting session key: %v", err) } m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae) + if err != nil { + utils.Fatalf("had an error generating the manifest: %v", err) + } if dryRun { err = printManifests(m, nil) if err != nil { @@ -130,7 +194,7 @@ func accessNewACT(ctx *cli.Context) { if err != nil { utils.Fatalf("had an error reading the grantee public key list") } - pkGrantees = strings.Split(string(bytes), "\n") + pkGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n") } if passGranteesFilename != "" { @@ -138,7 +202,7 @@ func accessNewACT(ctx *cli.Context) { if err != nil { utils.Fatalf("could not read password filename: %v", err) } - passGrantees = strings.Split(string(bytes), "\n") + passGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n") } accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees) if err != nil { diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 106e6dd912..9357c577e3 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -// +build !windows - package main import ( @@ -28,6 +26,7 @@ import ( gorand "math/rand" "net/http" "os" + "runtime" "strings" "testing" "time" @@ -37,7 +36,7 @@ import ( "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmapi "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -48,23 +47,41 @@ const ( var DefaultCurve = crypto.S256() -// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. +func TestACT(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + initCluster(t) + + cases := []struct { + name string + f func(t *testing.T) + }{ + {"Password", testPassword}, + {"PK", testPK}, + {"ACTWithoutBogus", testACTWithoutBogus}, + {"ACTWithBogus", testACTWithBogus}, + } + + for _, tc := range cases { + t.Run(tc.name, tc.f) + } +} + +// testPassword tests for the correct creation of an ACT manifest protected by a password. // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded // is then fetched through 2nd node. since the tested code is not key-aware - we can just // fetch from the 2nd node using HTTP BasicAuth -func TestAccessPassword(t *testing.T) { - cluster := newTestCluster(t, 1) - defer cluster.Shutdown() - proxyNode := cluster.Nodes[0] - +func testPassword(t *testing.T) { dataFilename := testutil.TempFileWithContent(t, data) defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, "--bzzapi", - proxyNode.URL, //it doesn't matter through which node we upload content + cluster.Nodes[0].URL, "up", "--encrypt", dataFilename) @@ -138,16 +155,17 @@ func TestAccessPassword(t *testing.T) { if a.Publisher != "" { t.Fatal("should be empty") } - client := swarm.NewClient(cluster.Nodes[0].URL) + + client := swarmapi.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) if err != nil { t.Fatal(err) } - httpClient := &http.Client{} - url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash + + httpClient := &http.Client{} response, err := httpClient.Get(url) if err != nil { t.Fatal(err) @@ -189,7 +207,7 @@ func TestAccessPassword(t *testing.T) { //download file with 'swarm down' with wrong password up = runSwarm(t, "--bzzapi", - proxyNode.URL, + cluster.Nodes[0].URL, "down", "bzz:/"+hash, tmp, @@ -203,16 +221,12 @@ func TestAccessPassword(t *testing.T) { up.ExpectExit() } -// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). +// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears. // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware, // the test will fail if the proxy's given private key is not granted on the ACT. -func TestAccessPK(t *testing.T) { - // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 2) - defer cluster.Shutdown() - +func testPK(t *testing.T) { dataFilename := testutil.TempFileWithContent(t, data) defer os.RemoveAll(dataFilename) @@ -318,7 +332,7 @@ func TestAccessPK(t *testing.T) { if a.Publisher != pkComp { t.Fatal("publisher key did not match") } - client := swarm.NewClient(cluster.Nodes[0].URL) + client := swarmapi.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) if err != nil { @@ -344,29 +358,24 @@ func TestAccessPK(t *testing.T) { } } -// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) -func TestAccessACT(t *testing.T) { - testAccessACT(t, 0) +// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) +func testACTWithoutBogus(t *testing.T) { + testACT(t, 0) } -// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest) -func TestAccessACTScale(t *testing.T) { - testAccessACT(t, 1000) +// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest) +func testACTWithBogus(t *testing.T) { + testACT(t, 100) } -// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection +// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access. // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails. // the publisher uploads through one of the nodes then disappears. -func testAccessACT(t *testing.T, bogusEntries int) { - // Setup Swarm and upload a test file to it - const clusterSize = 3 - cluster := newTestCluster(t, clusterSize) - defer cluster.Shutdown() - +func testACT(t *testing.T, bogusEntries int) { var uploadThroughNode = cluster.Nodes[0] - client := swarm.NewClient(uploadThroughNode.URL) + client := swarmapi.NewClient(uploadThroughNode.URL) r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index ae4b5816e6..3eea3057b3 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -59,27 +59,29 @@ var ( //constants for environment variables const ( - SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" - SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" - SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" - SWARM_ENV_PORT = "SWARM_PORT" - SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" - SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" - SWARM_ENV_SWAP_API = "SWARM_SWAP_API" - SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE" - SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY" - SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE" - SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK" - SWARM_ENV_ENS_API = "SWARM_ENS_API" - SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" - SWARM_ENV_CORS = "SWARM_CORS" - SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" - SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE" - SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" - SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" - SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" - SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" - GETH_ENV_DATADIR = "GETH_DATADIR" + SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" + SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" + SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" + SWARM_ENV_PORT = "SWARM_PORT" + SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" + SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" + SWARM_ENV_SWAP_API = "SWARM_SWAP_API" + SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE" + SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY" + SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS" + SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE" + SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK" + SWARM_ENV_ENS_API = "SWARM_ENS_API" + SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" + SWARM_ENV_CORS = "SWARM_CORS" + SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" + SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE" + SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" + SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" + SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" + SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" + GETH_ENV_DATADIR = "GETH_DATADIR" ) // These settings ensure that TOML keys use the same names as Go struct fields. @@ -124,7 +126,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) { //get the account for the provided swarm account prvkey := getAccount(config.BzzAccount, ctx, stack) //set the resolved config path (geth --datadir) - config.Path = stack.InstanceDir() + config.Path = expandPath(stack.InstanceDir()) //finally, initialize the configuration config.Init(prvkey) //configuration phase completed here @@ -175,14 +177,18 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con } if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" { - if id, _ := strconv.Atoi(networkid); id != 0 { - currentConfig.NetworkID = uint64(id) + id, err := strconv.ParseUint(networkid, 10, 64) + if err != nil { + utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err) + } + if id != 0 { + currentConfig.NetworkID = id } } if ctx.GlobalIsSet(utils.DataDirFlag.Name) { if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" { - currentConfig.Path = datadir + currentConfig.Path = expandPath(datadir) } } @@ -207,6 +213,9 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.SyncUpdateDelay = d } + // any value including 0 is acceptable + currentConfig.MaxStreamPeerServers = ctx.GlobalInt(SwarmMaxStreamPeerServersFlag.Name) + if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) { currentConfig.LightNodeEnabled = true } @@ -226,6 +235,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con if len(ensAPIs) == 1 && ensAPIs[0] == "" { ensAPIs = nil } + for i := range ensAPIs { + ensAPIs[i] = expandPath(ensAPIs[i]) + } + currentConfig.EnsAPIs = ensAPIs } @@ -262,13 +275,17 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" { - if id, _ := strconv.Atoi(networkid); id != 0 { - currentConfig.NetworkID = uint64(id) + id, err := strconv.ParseUint(networkid, 10, 64) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err) + } + if id != 0 { + currentConfig.NetworkID = id } } if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" { - currentConfig.Path = datadir + currentConfig.Path = expandPath(datadir) } bzzport := os.Getenv(SWARM_ENV_PORT) @@ -281,33 +298,50 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" { - if swap, err := strconv.ParseBool(swapenable); err != nil { - currentConfig.SwapEnabled = swap + swap, err := strconv.ParseBool(swapenable) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err) } + currentConfig.SwapEnabled = swap } if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" { - if sync, err := strconv.ParseBool(syncdisable); err != nil { - currentConfig.SyncEnabled = !sync + sync, err := strconv.ParseBool(syncdisable) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err) } + currentConfig.SyncEnabled = !sync } if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" { - if skipCheck, err := strconv.ParseBool(v); err != nil { + skipCheck, err := strconv.ParseBool(v) + if err != nil { currentConfig.DeliverySkipCheck = skipCheck } } if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" { - if d, err := time.ParseDuration(v); err != nil { - currentConfig.SyncUpdateDelay = d + d, err := time.ParseDuration(v) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err) } + currentConfig.SyncUpdateDelay = d + } + + if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" { + m, err := strconv.Atoi(max) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err) + } + currentConfig.MaxStreamPeerServers = m } if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" { - if lightnode, err := strconv.ParseBool(lne); err != nil { - currentConfig.LightNodeEnabled = lightnode + lightnode, err := strconv.ParseBool(lne) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err) } + currentConfig.LightNodeEnabled = lightnode } if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" { diff --git a/cmd/swarm/db.go b/cmd/swarm/db.go index fe03f2d160..7916beffce 100644 --- a/cmd/swarm/db.go +++ b/cmd/swarm/db.go @@ -29,6 +29,48 @@ import ( "gopkg.in/urfave/cli.v1" ) +var dbCommand = cli.Command{ + Name: "db", + CustomHelpTemplate: helpTemplate, + Usage: "manage the local chunk database", + ArgsUsage: "db COMMAND", + Description: "Manage the local chunk database", + Subcommands: []cli.Command{ + { + Action: dbExport, + CustomHelpTemplate: helpTemplate, + Name: "export", + Usage: "export a local chunk database as a tar archive (use - to send to stdout)", + ArgsUsage: " ", + Description: ` +Export a local chunk database as a tar archive (use - to send to stdout). + + swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar + +The export may be quite large, consider piping the output through the Unix +pv(1) tool to get a progress bar: + + swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar +`, + }, + { + Action: dbImport, + CustomHelpTemplate: helpTemplate, + Name: "import", + Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)", + ArgsUsage: " ", + Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin). + + swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar + +The import may be quite large, consider piping the input through the Unix +pv(1) tool to get a progress bar: + + pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, + }, + }, +} + func dbExport(ctx *cli.Context) { args := ctx.Args() if len(args) != 3 { @@ -93,21 +135,6 @@ func dbImport(ctx *cli.Context) { log.Info(fmt.Sprintf("successfully imported %d chunks", count)) } -func dbClean(ctx *cli.Context) { - args := ctx.Args() - if len(args) != 2 { - utils.Fatalf("invalid arguments, please specify (path to a local chunk database) and the base key") - } - - store, err := openLDBStore(args[0], common.Hex2Bytes(args[1])) - if err != nil { - utils.Fatalf("error opening local chunk database: %s", err) - } - defer store.Close() - - store.Cleanup() -} - func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { return nil, fmt.Errorf("invalid chunkdb path: %s", err) diff --git a/cmd/swarm/download.go b/cmd/swarm/download.go index 91bc2c93ab..fcbefa0206 100644 --- a/cmd/swarm/download.go +++ b/cmd/swarm/download.go @@ -28,6 +28,15 @@ import ( "gopkg.in/urfave/cli.v1" ) +var downloadCommand = cli.Command{ + Action: download, + Name: "down", + Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag}, + Usage: "downloads a swarm manifest or a file inside a manifest", + ArgsUsage: " []", + Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`, +} + func download(ctx *cli.Context) { log.Debug("downloading content using swarm down") args := ctx.Args() diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go index 525538ad75..e8671eea79 100644 --- a/cmd/swarm/export_test.go +++ b/cmd/swarm/export_test.go @@ -19,15 +19,15 @@ package main import ( "bytes" "crypto/md5" - "crypto/rand" "io" - "io/ioutil" "net/http" "os" + "runtime" "strings" "testing" "github.com/ethereum/go-ethereum/swarm" + "github.com/ethereum/go-ethereum/swarm/testutil" ) // TestCLISwarmExportImport perform the following test: @@ -38,14 +38,18 @@ import ( // 5. imports the exported datastore // 6. fetches the uploaded random file from the second node func TestCLISwarmExportImport(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } cluster := newTestCluster(t, 1) - // generate random 10mb file - f, cleanup := generateRandomFile(t, 10000000) - defer cleanup() + // generate random 1mb file + content := testutil.RandomBytes(1, 1000000) + fileName := testutil.TempFileWithContent(t, string(content)) + defer os.Remove(fileName) // upload the file with 'swarm up' and expect a hash - up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name()) + up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName) _, matches := up.ExpectRegexp(`[a-f\d]{64}`) up.ExpectExit() hash := matches[0] @@ -92,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) { } // compare downloaded file with the generated random file - mustEqualFiles(t, f, res.Body) + mustEqualFiles(t, bytes.NewReader(content), res.Body) } func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { @@ -113,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen) } } - -func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) { - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - - // callback for tmp file cleanup - teardown = func() { - tmp.Close() - os.Remove(tmp.Name()) - } - - // write 10mb random data to file - buf := make([]byte, 10000000) - _, err = rand.Read(buf) - if err != nil { - t.Fatal(err) - } - ioutil.WriteFile(tmp.Name(), buf, 0755) - - return tmp, teardown -} diff --git a/cmd/swarm/feeds.go b/cmd/swarm/feeds.go new file mode 100644 index 0000000000..f26a8cc7dc --- /dev/null +++ b/cmd/swarm/feeds.go @@ -0,0 +1,234 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Command feed allows the user to create and update signed Swarm feeds +package main + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + + "github.com/ethereum/go-ethereum/cmd/utils" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "gopkg.in/urfave/cli.v1" +) + +var feedCommand = cli.Command{ + CustomHelpTemplate: helpTemplate, + Name: "feed", + Usage: "(Advanced) Create and update Swarm Feeds", + ArgsUsage: "", + Description: "Works with Swarm Feeds", + Subcommands: []cli.Command{ + { + Action: feedCreateManifest, + CustomHelpTemplate: helpTemplate, + Name: "create", + Usage: "creates and publishes a new feed manifest", + Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic. + The feed topic can be built in the following ways: + * use --topic to set the topic to an arbitrary binary hex string. + * use --name to set the topic to a human-readable name. + For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. + * use both --topic and --name to create named subtopics. + For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning + this feed tracks a discussion about that contract. + The --user flag allows to have this manifest refer to a user other than yourself. If not specified, + it will then default to your local account (--bzzaccount)`, + Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, + }, + { + Action: feedUpdate, + CustomHelpTemplate: helpTemplate, + Name: "update", + Usage: "updates the content of an existing Swarm Feed", + ArgsUsage: "<0x Hex data>", + Description: `publishes a new update on the specified topic + The feed topic can be built in the following ways: + * use --topic to set the topic to an arbitrary binary hex string. + * use --name to set the topic to a human-readable name. + For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. + * use both --topic and --name to create named subtopics. + For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning + this feed tracks a discussion about that contract. + + If you have a manifest, you can specify it with --manifest to refer to the feed, + instead of using --topic / --name + `, + Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag}, + }, + { + Action: feedInfo, + CustomHelpTemplate: helpTemplate, + Name: "info", + Usage: "obtains information about an existing Swarm feed", + Description: `obtains information about an existing Swarm feed + The topic can be specified directly with the --topic flag as an hex string + If no topic is specified, the default topic (zero) will be used + The --name flag can be used to specify subtopics with a specific name. + The --user flag allows to refer to a user other than yourself. If not specified, + it will then default to your local account (--bzzaccount) + If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user + to refer to the feed`, + Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, + }, + }, +} + +func NewGenericSigner(ctx *cli.Context) feed.Signer { + return feed.NewGenericSigner(getPrivKey(ctx)) +} + +func getTopic(ctx *cli.Context) (topic feed.Topic) { + var name = ctx.String(SwarmFeedNameFlag.Name) + var relatedTopic = ctx.String(SwarmFeedTopicFlag.Name) + var relatedTopicBytes []byte + var err error + + if relatedTopic != "" { + relatedTopicBytes, err = hexutil.Decode(relatedTopic) + if err != nil { + utils.Fatalf("Error parsing topic: %s", err) + } + } + + topic, err = feed.NewTopic(name, relatedTopicBytes) + if err != nil { + utils.Fatalf("Error parsing topic: %s", err) + } + return topic +} + +// swarm feed create [--name ] [--data <0x Hexdata> [--multihash=false]] +// swarm feed update <0x Hexdata> [--multihash=false] +// swarm feed info + +func feedCreateManifest(ctx *cli.Context) { + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = swarm.NewClient(bzzapi) + ) + + newFeedUpdateRequest := feed.NewFirstRequest(getTopic(ctx)) + newFeedUpdateRequest.Feed.User = feedGetUser(ctx) + + manifestAddress, err := client.CreateFeedWithManifest(newFeedUpdateRequest) + if err != nil { + utils.Fatalf("Error creating feed manifest: %s", err.Error()) + return + } + fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up) + +} + +func feedUpdate(ctx *cli.Context) { + args := ctx.Args() + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = swarm.NewClient(bzzapi) + manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name) + ) + + if len(args) < 1 { + fmt.Println("Incorrect number of arguments") + cli.ShowCommandHelpAndExit(ctx, "update", 1) + return + } + + signer := NewGenericSigner(ctx) + + data, err := hexutil.Decode(args[0]) + if err != nil { + utils.Fatalf("Error parsing data: %s", err.Error()) + return + } + + var updateRequest *feed.Request + var query *feed.Query + + if manifestAddressOrDomain == "" { + query = new(feed.Query) + query.User = signer.Address() + query.Topic = getTopic(ctx) + + } + + // Retrieve a feed update request + updateRequest, err = client.GetFeedRequest(query, manifestAddressOrDomain) + if err != nil { + utils.Fatalf("Error retrieving feed status: %s", err.Error()) + } + + // set the new data + updateRequest.SetData(data) + + // sign update + if err = updateRequest.Sign(signer); err != nil { + utils.Fatalf("Error signing feed update: %s", err.Error()) + } + + // post update + err = client.UpdateFeed(updateRequest) + if err != nil { + utils.Fatalf("Error updating feed: %s", err.Error()) + return + } +} + +func feedInfo(ctx *cli.Context) { + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = swarm.NewClient(bzzapi) + manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name) + ) + + var query *feed.Query + if manifestAddressOrDomain == "" { + query = new(feed.Query) + query.Topic = getTopic(ctx) + query.User = feedGetUser(ctx) + } + + metadata, err := client.GetFeedRequest(query, manifestAddressOrDomain) + if err != nil { + utils.Fatalf("Error retrieving feed metadata: %s", err.Error()) + return + } + encodedMetadata, err := metadata.MarshalJSON() + if err != nil { + utils.Fatalf("Error encoding metadata to JSON for display:%s", err) + } + fmt.Println(string(encodedMetadata)) +} + +func feedGetUser(ctx *cli.Context) common.Address { + var user = ctx.String(SwarmFeedUserFlag.Name) + if user != "" { + return common.HexToAddress(user) + } + pk := getPrivKey(ctx) + if pk == nil { + utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount") + } + return crypto.PubkeyToAddress(pk.PublicKey) + +} diff --git a/cmd/swarm/feeds_test.go b/cmd/swarm/feeds_test.go new file mode 100644 index 0000000000..a0cedf0d36 --- /dev/null +++ b/cmd/swarm/feeds_test.go @@ -0,0 +1,165 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +func TestCLIFeedUpdate(t *testing.T) { + srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer { + return swarmhttp.NewServer(api, "") + }, nil) + log.Info("starting a test swarm server") + defer srv.Close() + + // create a private key file for signing + privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979" + privKey, _ := crypto.HexToECDSA(privkeyHex) + address := crypto.PubkeyToAddress(privKey.PublicKey) + + pkFileName := testutil.TempFileWithContent(t, privkeyHex) + defer os.Remove(pkFileName) + + // compose a topic. We'll be doing quotes about Miguel de Cervantes + var topic feed.Topic + subject := []byte("Miguel de Cervantes") + copy(topic[:], subject[:]) + name := "quotes" + + // prepare some data for the update + data := []byte("En boca cerrada no entran moscas") + hexData := hexutil.Encode(data) + + flags := []string{ + "--bzzapi", srv.URL, + "--bzzaccount", pkFileName, + "feed", "update", + "--topic", topic.Hex(), + "--name", name, + hexData} + + // create an update and expect an exit without errors + log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'")) + cmd := runSwarm(t, flags...) + cmd.ExpectExit() + + // now try to get the update using the client + client := swarm.NewClient(srv.URL) + + // build the same topic as before, this time + // we use NewTopic to create a topic automatically. + topic, err := feed.NewTopic(name, subject) + if err != nil { + t.Fatal(err) + } + + // Feed configures whose updates we will be looking up. + fd := feed.Feed{ + Topic: topic, + User: address, + } + + // Build a query to get the latest update + query := feed.NewQueryLatest(&fd, lookup.NoClue) + + // retrieve content! + reader, err := client.QueryFeed(query, "") + if err != nil { + t.Fatal(err) + } + + retrieved, err := ioutil.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + // check we retrieved the sent information + if !bytes.Equal(data, retrieved) { + t.Fatalf("Received %s, expected %s", retrieved, data) + } + + // Now retrieve info for the next update + flags = []string{ + "--bzzapi", srv.URL, + "feed", "info", + "--topic", topic.Hex(), + "--user", address.Hex(), + } + + log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'")) + cmd = runSwarm(t, flags...) + _, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout + cmd.ExpectExit() + + // verify we can deserialize the result as a valid JSON + var request feed.Request + err = json.Unmarshal([]byte(matches[0]), &request) + if err != nil { + t.Fatal(err) + } + + // make sure the retrieved feed is the same + if request.Feed != fd { + t.Fatalf("Expected feed to be: %s, got %s", fd, request.Feed) + } + + // test publishing a manifest + flags = []string{ + "--bzzapi", srv.URL, + "--bzzaccount", pkFileName, + "feed", "create", + "--topic", topic.Hex(), + } + + log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'")) + cmd = runSwarm(t, flags...) + _, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout + cmd.ExpectExit() + + manifestAddress := matches[0] // read the received feed manifest + + // now attempt to lookup the latest update using a manifest instead + reader, err = client.QueryFeed(nil, manifestAddress) + if err != nil { + t.Fatal(err) + } + + retrieved, err = ioutil.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, retrieved) { + t.Fatalf("Received %s, expected %s", retrieved, data) + } +} diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go new file mode 100644 index 0000000000..0dedca674b --- /dev/null +++ b/cmd/swarm/flags.go @@ -0,0 +1,179 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Command feed allows the user to create and update signed Swarm feeds +package main + +import cli "gopkg.in/urfave/cli.v1" + +var ( + ChequebookAddrFlag = cli.StringFlag{ + Name: "chequebook", + Usage: "chequebook contract address", + EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR, + } + SwarmAccountFlag = cli.StringFlag{ + Name: "bzzaccount", + Usage: "Swarm account key file", + EnvVar: SWARM_ENV_ACCOUNT, + } + SwarmListenAddrFlag = cli.StringFlag{ + Name: "httpaddr", + Usage: "Swarm HTTP API listening interface", + EnvVar: SWARM_ENV_LISTEN_ADDR, + } + SwarmPortFlag = cli.StringFlag{ + Name: "bzzport", + Usage: "Swarm local http api port", + EnvVar: SWARM_ENV_PORT, + } + SwarmNetworkIdFlag = cli.IntFlag{ + Name: "bzznetworkid", + Usage: "Network identifier (integer, default 3=swarm testnet)", + EnvVar: SWARM_ENV_NETWORK_ID, + } + SwarmSwapEnabledFlag = cli.BoolFlag{ + Name: "swap", + Usage: "Swarm SWAP enabled (default false)", + EnvVar: SWARM_ENV_SWAP_ENABLE, + } + SwarmSwapAPIFlag = cli.StringFlag{ + Name: "swap-api", + Usage: "URL of the Ethereum API provider to use to settle SWAP payments", + EnvVar: SWARM_ENV_SWAP_API, + } + SwarmSyncDisabledFlag = cli.BoolTFlag{ + Name: "nosync", + Usage: "Disable swarm syncing", + EnvVar: SWARM_ENV_SYNC_DISABLE, + } + SwarmSyncUpdateDelay = cli.DurationFlag{ + Name: "sync-update-delay", + Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", + EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, + } + SwarmMaxStreamPeerServersFlag = cli.IntFlag{ + Name: "max-stream-peer-servers", + Usage: "Limit of Stream peer servers, 0 denotes unlimited", + EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS, + Value: 10000, // A very large default value is possible as stream servers have very small memory footprint + } + SwarmLightNodeEnabled = cli.BoolFlag{ + Name: "lightnode", + Usage: "Enable Swarm LightNode (default false)", + EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE, + } + SwarmDeliverySkipCheckFlag = cli.BoolFlag{ + Name: "delivery-skip-check", + Usage: "Skip chunk delivery check (default false)", + EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK, + } + EnsAPIFlag = cli.StringSliceFlag{ + Name: "ens-api", + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", + EnvVar: SWARM_ENV_ENS_API, + } + SwarmApiFlag = cli.StringFlag{ + Name: "bzzapi", + Usage: "Specifies the Swarm HTTP endpoint to connect to", + Value: "http://127.0.0.1:8500", + } + SwarmRecursiveFlag = cli.BoolFlag{ + Name: "recursive", + Usage: "Upload directories recursively", + } + SwarmWantManifestFlag = cli.BoolTFlag{ + Name: "manifest", + Usage: "Automatic manifest upload (default true)", + } + SwarmUploadDefaultPath = cli.StringFlag{ + Name: "defaultpath", + Usage: "path to file served for empty url path (none)", + } + SwarmAccessGrantKeyFlag = cli.StringFlag{ + Name: "grant-key", + Usage: "grants a given public key access to an ACT", + } + SwarmAccessGrantKeysFlag = cli.StringFlag{ + Name: "grant-keys", + Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT", + } + SwarmUpFromStdinFlag = cli.BoolFlag{ + Name: "stdin", + Usage: "reads data to be uploaded from stdin", + } + SwarmUploadMimeType = cli.StringFlag{ + Name: "mime", + Usage: "Manually specify MIME type", + } + SwarmEncryptedFlag = cli.BoolFlag{ + Name: "encrypt", + Usage: "use encrypted upload", + } + SwarmAccessPasswordFlag = cli.StringFlag{ + Name: "password", + Usage: "Password", + EnvVar: SWARM_ACCESS_PASSWORD, + } + SwarmDryRunFlag = cli.BoolFlag{ + Name: "dry-run", + Usage: "dry-run", + } + CorsStringFlag = cli.StringFlag{ + Name: "corsdomain", + Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", + EnvVar: SWARM_ENV_CORS, + } + SwarmStorePath = cli.StringFlag{ + Name: "store.path", + Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)", + EnvVar: SWARM_ENV_STORE_PATH, + } + SwarmStoreCapacity = cli.Uint64Flag{ + Name: "store.size", + Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)", + EnvVar: SWARM_ENV_STORE_CAPACITY, + } + SwarmStoreCacheCapacity = cli.UintFlag{ + Name: "store.cache.size", + Usage: "Number of recent chunks cached in memory (default 5000)", + EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, + } + SwarmCompressedFlag = cli.BoolFlag{ + Name: "compressed", + Usage: "Prints encryption keys in compressed form", + } + SwarmFeedNameFlag = cli.StringFlag{ + Name: "name", + Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", + } + SwarmFeedTopicFlag = cli.StringFlag{ + Name: "topic", + Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters", + } + SwarmFeedDataOnCreateFlag = cli.StringFlag{ + Name: "data", + Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x", + } + SwarmFeedManifestFlag = cli.StringFlag{ + Name: "manifest", + Usage: "Refers to the feed through a manifest", + } + SwarmFeedUserFlag = cli.StringFlag{ + Name: "user", + Usage: "Indicates the user who updates the feed", + } +) diff --git a/cmd/swarm/fs.go b/cmd/swarm/fs.go index 3dc38ca4da..b970b2e8c2 100644 --- a/cmd/swarm/fs.go +++ b/cmd/swarm/fs.go @@ -30,6 +30,43 @@ import ( "gopkg.in/urfave/cli.v1" ) +var fsCommand = cli.Command{ + Name: "fs", + CustomHelpTemplate: helpTemplate, + Usage: "perform FUSE operations", + ArgsUsage: "fs COMMAND", + Description: "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node", + Subcommands: []cli.Command{ + { + Action: mount, + CustomHelpTemplate: helpTemplate, + Name: "mount", + Flags: []cli.Flag{utils.IPCPathFlag}, + Usage: "mount a swarm hash to a mount point", + ArgsUsage: "swarm fs mount --ipcpath ", + Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", + }, + { + Action: unmount, + CustomHelpTemplate: helpTemplate, + Name: "unmount", + Flags: []cli.Flag{utils.IPCPathFlag}, + Usage: "unmount a swarmfs mount", + ArgsUsage: "swarm fs unmount --ipcpath ", + Description: "Unmounts a swarmfs mount residing at . This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", + }, + { + Action: listMounts, + CustomHelpTemplate: helpTemplate, + Name: "list", + Flags: []cli.Flag{utils.IPCPathFlag}, + Usage: "list swarmfs mounts", + ArgsUsage: "swarm fs list --ipcpath ", + Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", + }, + }, +} + func mount(cliContext *cli.Context) { args := cliContext.Args() if len(args) < 2 { diff --git a/cmd/swarm/fs_test.go b/cmd/swarm/fs_test.go index 4f38b094bb..ac4223b661 100644 --- a/cmd/swarm/fs_test.go +++ b/cmd/swarm/fs_test.go @@ -29,14 +29,8 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" ) -func init() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) -} - type testFile struct { filePath string content string @@ -80,6 +74,9 @@ func TestCLISwarmFs(t *testing.T) { t.Fatal(err) } dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir") + if err != nil { + t.Fatal(err) + } dummyContent := "somerandomtestcontentthatshouldbeasserted" dirs := []string{ diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index d679806e38..471feb53d6 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -27,6 +27,15 @@ import ( "gopkg.in/urfave/cli.v1" ) +var hashCommand = cli.Command{ + Action: hash, + CustomHelpTemplate: helpTemplate, + Name: "hash", + Usage: "print the swarm hash of a file or directory", + ArgsUsage: "", + Description: "Prints the swarm hash of file or directory", +} + func hash(ctx *cli.Context) { args := ctx.Args() if len(args) < 1 { diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go index 01b3f4ab6c..5d35154a57 100644 --- a/cmd/swarm/list.go +++ b/cmd/swarm/list.go @@ -27,6 +27,15 @@ import ( "gopkg.in/urfave/cli.v1" ) +var listCommand = cli.Command{ + Action: list, + CustomHelpTemplate: helpTemplate, + Name: "ls", + Usage: "list files and directories contained in a manifest", + ArgsUsage: " []", + Description: "Lists files and directories contained in a manifest", +} + func list(ctx *cli.Context) { args := ctx.Args() diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 4c9ce931ee..ccbb24eec3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -70,151 +70,6 @@ var ( gitCommit string // Git SHA1 commit hash of the release (set via linker flags) ) -var ( - ChequebookAddrFlag = cli.StringFlag{ - Name: "chequebook", - Usage: "chequebook contract address", - EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR, - } - SwarmAccountFlag = cli.StringFlag{ - Name: "bzzaccount", - Usage: "Swarm account key file", - EnvVar: SWARM_ENV_ACCOUNT, - } - SwarmListenAddrFlag = cli.StringFlag{ - Name: "httpaddr", - Usage: "Swarm HTTP API listening interface", - EnvVar: SWARM_ENV_LISTEN_ADDR, - } - SwarmPortFlag = cli.StringFlag{ - Name: "bzzport", - Usage: "Swarm local http api port", - EnvVar: SWARM_ENV_PORT, - } - SwarmNetworkIdFlag = cli.IntFlag{ - Name: "bzznetworkid", - Usage: "Network identifier (integer, default 3=swarm testnet)", - EnvVar: SWARM_ENV_NETWORK_ID, - } - SwarmSwapEnabledFlag = cli.BoolFlag{ - Name: "swap", - Usage: "Swarm SWAP enabled (default false)", - EnvVar: SWARM_ENV_SWAP_ENABLE, - } - SwarmSwapAPIFlag = cli.StringFlag{ - Name: "swap-api", - Usage: "URL of the Ethereum API provider to use to settle SWAP payments", - EnvVar: SWARM_ENV_SWAP_API, - } - SwarmSyncDisabledFlag = cli.BoolTFlag{ - Name: "nosync", - Usage: "Disable swarm syncing", - EnvVar: SWARM_ENV_SYNC_DISABLE, - } - SwarmSyncUpdateDelay = cli.DurationFlag{ - Name: "sync-update-delay", - Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", - EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, - } - SwarmLightNodeEnabled = cli.BoolFlag{ - Name: "lightnode", - Usage: "Enable Swarm LightNode (default false)", - EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE, - } - SwarmDeliverySkipCheckFlag = cli.BoolFlag{ - Name: "delivery-skip-check", - Usage: "Skip chunk delivery check (default false)", - EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK, - } - EnsAPIFlag = cli.StringSliceFlag{ - Name: "ens-api", - Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", - EnvVar: SWARM_ENV_ENS_API, - } - SwarmApiFlag = cli.StringFlag{ - Name: "bzzapi", - Usage: "Swarm HTTP endpoint", - Value: "http://127.0.0.1:8500", - } - SwarmRecursiveFlag = cli.BoolFlag{ - Name: "recursive", - Usage: "Upload directories recursively", - } - SwarmWantManifestFlag = cli.BoolTFlag{ - Name: "manifest", - Usage: "Automatic manifest upload (default true)", - } - SwarmUploadDefaultPath = cli.StringFlag{ - Name: "defaultpath", - Usage: "path to file served for empty url path (none)", - } - SwarmAccessGrantKeyFlag = cli.StringFlag{ - Name: "grant-key", - Usage: "grants a given public key access to an ACT", - } - SwarmAccessGrantKeysFlag = cli.StringFlag{ - Name: "grant-keys", - Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT", - } - SwarmUpFromStdinFlag = cli.BoolFlag{ - Name: "stdin", - Usage: "reads data to be uploaded from stdin", - } - SwarmUploadMimeType = cli.StringFlag{ - Name: "mime", - Usage: "Manually specify MIME type", - } - SwarmEncryptedFlag = cli.BoolFlag{ - Name: "encrypt", - Usage: "use encrypted upload", - } - SwarmAccessPasswordFlag = cli.StringFlag{ - Name: "password", - Usage: "Password", - EnvVar: SWARM_ACCESS_PASSWORD, - } - SwarmDryRunFlag = cli.BoolFlag{ - Name: "dry-run", - Usage: "dry-run", - } - CorsStringFlag = cli.StringFlag{ - Name: "corsdomain", - Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", - EnvVar: SWARM_ENV_CORS, - } - SwarmStorePath = cli.StringFlag{ - Name: "store.path", - Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)", - EnvVar: SWARM_ENV_STORE_PATH, - } - SwarmStoreCapacity = cli.Uint64Flag{ - Name: "store.size", - Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)", - EnvVar: SWARM_ENV_STORE_CAPACITY, - } - SwarmStoreCacheCapacity = cli.UintFlag{ - Name: "store.cache.size", - Usage: "Number of recent chunks cached in memory (default 5000)", - EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, - } - SwarmResourceMultihashFlag = cli.BoolFlag{ - Name: "multihash", - Usage: "Determines how to interpret data for a resource update. If not present, data will be interpreted as raw, literal data that will be included in the resource", - } - SwarmResourceNameFlag = cli.StringFlag{ - Name: "name", - Usage: "User-defined name for the new resource", - } - SwarmResourceDataOnCreateFlag = cli.StringFlag{ - Name: "data", - Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x", - } - SwarmCompressedFlag = cli.BoolFlag{ - Name: "compressed", - Usage: "Prints encryption keys in compressed form", - } -) - //declare a few constant error messages, useful for later error check comparisons in test var ( SWARM_ERR_NO_BZZACCOUNT = "bzzaccount option is required but not set; check your config file, command line or environment variables" @@ -242,12 +97,12 @@ func init() { utils.ListenPortFlag.Value = 30399 } -var app = utils.NewApp(gitCommit, "Ethereum Swarm") +var app = utils.NewApp("", "Ethereum Swarm") // This init function creates the cli.App. func init() { app.Action = bzzd - app.HideVersion = true // we have a command to print the version + app.Version = sv.ArchiveVersion(gitCommit) app.Copyright = "Copyright 2013-2016 The go-ethereum Authors" app.Commands = []cli.Command{ { @@ -265,249 +120,24 @@ func init() { Usage: "Print public key information", Description: "The output of this command is supposed to be machine-readable", }, - { - Action: upload, - CustomHelpTemplate: helpTemplate, - Name: "up", - Usage: "uploads a file or directory to swarm using the HTTP API", - ArgsUsage: "", - Flags: []cli.Flag{SwarmEncryptedFlag}, - Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash", - }, - { - CustomHelpTemplate: helpTemplate, - Name: "access", - Usage: "encrypts a reference and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root manifest", - Subcommands: []cli.Command{ - { - CustomHelpTemplate: helpTemplate, - Name: "new", - Usage: "encrypts a reference and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - Subcommands: []cli.Command{ - { - Action: accessNewPass, - CustomHelpTemplate: helpTemplate, - Flags: []cli.Flag{ - utils.PasswordFileFlag, - SwarmDryRunFlag, - }, - Name: "pass", - Usage: "encrypts a reference with a password and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - }, - { - Action: accessNewPK, - CustomHelpTemplate: helpTemplate, - Flags: []cli.Flag{ - utils.PasswordFileFlag, - SwarmDryRunFlag, - SwarmAccessGrantKeyFlag, - }, - Name: "pk", - Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - }, - { - Action: accessNewACT, - CustomHelpTemplate: helpTemplate, - Flags: []cli.Flag{ - SwarmAccessGrantKeysFlag, - SwarmDryRunFlag, - utils.PasswordFileFlag, - }, - Name: "act", - Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", - ArgsUsage: "", - Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", - }, - }, - }, - }, - }, - { - CustomHelpTemplate: helpTemplate, - Name: "resource", - Usage: "(Advanced) Create and update Mutable Resources", - ArgsUsage: "", - Description: "Works with Mutable Resource Updates", - Subcommands: []cli.Command{ - { - Action: resourceCreate, - CustomHelpTemplate: helpTemplate, - Name: "create", - Usage: "creates a new Mutable Resource", - ArgsUsage: "", - Description: "creates a new Mutable Resource", - Flags: []cli.Flag{SwarmResourceNameFlag, SwarmResourceDataOnCreateFlag, SwarmResourceMultihashFlag}, - }, - { - Action: resourceUpdate, - CustomHelpTemplate: helpTemplate, - Name: "update", - Usage: "updates the content of an existing Mutable Resource", - ArgsUsage: " <0x Hex data>", - Description: "updates the content of an existing Mutable Resource", - Flags: []cli.Flag{SwarmResourceMultihashFlag}, - }, - { - Action: resourceInfo, - CustomHelpTemplate: helpTemplate, - Name: "info", - Usage: "obtains information about an existing Mutable Resource", - ArgsUsage: "", - Description: "obtains information about an existing Mutable Resource", - }, - }, - }, - { - Action: list, - CustomHelpTemplate: helpTemplate, - Name: "ls", - Usage: "list files and directories contained in a manifest", - ArgsUsage: " []", - Description: "Lists files and directories contained in a manifest", - }, - { - Action: hash, - CustomHelpTemplate: helpTemplate, - Name: "hash", - Usage: "print the swarm hash of a file or directory", - ArgsUsage: "", - Description: "Prints the swarm hash of file or directory", - }, - { - Action: download, - Name: "down", - Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag}, - Usage: "downloads a swarm manifest or a file inside a manifest", - ArgsUsage: " []", - Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`, - }, - { - Name: "manifest", - CustomHelpTemplate: helpTemplate, - Usage: "perform operations on swarm manifests", - ArgsUsage: "COMMAND", - Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", - Subcommands: []cli.Command{ - { - Action: manifestAdd, - CustomHelpTemplate: helpTemplate, - Name: "add", - Usage: "add a new path to the manifest", - ArgsUsage: " ", - Description: "Adds a new path to the manifest", - }, - { - Action: manifestUpdate, - CustomHelpTemplate: helpTemplate, - Name: "update", - Usage: "update the hash for an already existing path in the manifest", - ArgsUsage: " ", - Description: "Update the hash for an already existing path in the manifest", - }, - { - Action: manifestRemove, - CustomHelpTemplate: helpTemplate, - Name: "remove", - Usage: "removes a path from the manifest", - ArgsUsage: " ", - Description: "Removes a path from the manifest", - }, - }, - }, - { - Name: "fs", - CustomHelpTemplate: helpTemplate, - Usage: "perform FUSE operations", - ArgsUsage: "fs COMMAND", - Description: "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node", - Subcommands: []cli.Command{ - { - Action: mount, - CustomHelpTemplate: helpTemplate, - Name: "mount", - Flags: []cli.Flag{utils.IPCPathFlag}, - Usage: "mount a swarm hash to a mount point", - ArgsUsage: "swarm fs mount --ipcpath ", - Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", - }, - { - Action: unmount, - CustomHelpTemplate: helpTemplate, - Name: "unmount", - Flags: []cli.Flag{utils.IPCPathFlag}, - Usage: "unmount a swarmfs mount", - ArgsUsage: "swarm fs unmount --ipcpath ", - Description: "Unmounts a swarmfs mount residing at . This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", - }, - { - Action: listMounts, - CustomHelpTemplate: helpTemplate, - Name: "list", - Flags: []cli.Flag{utils.IPCPathFlag}, - Usage: "list swarmfs mounts", - ArgsUsage: "swarm fs list --ipcpath ", - Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file", - }, - }, - }, - { - Name: "db", - CustomHelpTemplate: helpTemplate, - Usage: "manage the local chunk database", - ArgsUsage: "db COMMAND", - Description: "Manage the local chunk database", - Subcommands: []cli.Command{ - { - Action: dbExport, - CustomHelpTemplate: helpTemplate, - Name: "export", - Usage: "export a local chunk database as a tar archive (use - to send to stdout)", - ArgsUsage: " ", - Description: ` -Export a local chunk database as a tar archive (use - to send to stdout). - - swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar - -The export may be quite large, consider piping the output through the Unix -pv(1) tool to get a progress bar: - - swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar -`, - }, - { - Action: dbImport, - CustomHelpTemplate: helpTemplate, - Name: "import", - Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)", - ArgsUsage: " ", - Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin). - - swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar - -The import may be quite large, consider piping the input through the Unix -pv(1) tool to get a progress bar: - - pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, - }, - { - Action: dbClean, - CustomHelpTemplate: helpTemplate, - Name: "clean", - Usage: "remove corrupt entries from a local chunk database", - ArgsUsage: "", - Description: "Remove corrupt entries from a local chunk database", - }, - }, - }, - + // See upload.go + upCommand, + // See access.go + accessCommand, + // See feeds.go + feedCommand, + // See list.go + listCommand, + // See hash.go + hashCommand, + // See download.go + downloadCommand, + // See manifest.go + manifestCommand, + // See fs.go + fsCommand, + // See db.go + dbCommand, // See config.go DumpConfigCommand, } @@ -542,6 +172,7 @@ pv(1) tool to get a progress bar: SwarmSwapAPIFlag, SwarmSyncDisabledFlag, SwarmSyncUpdateDelay, + SwarmMaxStreamPeerServersFlag, SwarmLightNodeEnabled, SwarmDeliverySkipCheckFlag, SwarmListenAddrFlag, @@ -697,7 +328,7 @@ func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.Pr } // getPrivKey returns the private key of the specified bzzaccount -// Used only by client commands, such as `resource` +// Used only by client commands, such as `feed` func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey { // booting up the swarm node just as we do in bzzd action bzzconfig, err := buildConfig(ctx) diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 0216ffc1dd..312c72fa21 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -28,6 +28,40 @@ import ( "gopkg.in/urfave/cli.v1" ) +var manifestCommand = cli.Command{ + Name: "manifest", + CustomHelpTemplate: helpTemplate, + Usage: "perform operations on swarm manifests", + ArgsUsage: "COMMAND", + Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", + Subcommands: []cli.Command{ + { + Action: manifestAdd, + CustomHelpTemplate: helpTemplate, + Name: "add", + Usage: "add a new path to the manifest", + ArgsUsage: " ", + Description: "Adds a new path to the manifest", + }, + { + Action: manifestUpdate, + CustomHelpTemplate: helpTemplate, + Name: "update", + Usage: "update the hash for an already existing path in the manifest", + ArgsUsage: " ", + Description: "Update the hash for an already existing path in the manifest", + }, + { + Action: manifestRemove, + CustomHelpTemplate: helpTemplate, + Name: "remove", + Usage: "removes a path from the manifest", + ArgsUsage: " ", + Description: "Removes a path from the manifest", + }, + }, +} + // manifestAdd adds a new entry to the manifest at the given path. // New entry hash, the last argument, must be the hash of a manifest // with only one entry, which meta-data will be added to the original manifest. diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go index 08fe0b2eb7..01d982aa7a 100644 --- a/cmd/swarm/manifest_test.go +++ b/cmd/swarm/manifest_test.go @@ -21,21 +21,31 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "testing" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" ) // TestManifestChange tests manifest add, update and remove // cli commands without encryption. func TestManifestChange(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testManifestChange(t, false) } // TestManifestChange tests manifest add, update and remove // cli commands with encryption enabled. func TestManifestChangeEncrypted(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testManifestChange(t, true) } @@ -48,8 +58,8 @@ func TestManifestChangeEncrypted(t *testing.T) { // Argument encrypt controls whether to use encryption or not. func testManifestChange(t *testing.T, encrypt bool) { t.Parallel() - cluster := newTestCluster(t, 1) - defer cluster.Shutdown() + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) + defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-manifest-test") if err != nil { @@ -85,7 +95,7 @@ func testManifestChange(t *testing.T, encrypt bool) { args := []string{ "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "--recursive", "--defaultpath", indexDataFilename, @@ -100,7 +110,7 @@ func testManifestChange(t *testing.T, encrypt bool) { checkHashLength(t, origManifestHash, encrypt) - client := swarm.NewClient(cluster.Nodes[0].URL) + client := swarm.NewClient(srv.URL) // upload a new file and use its manifest to add it the original manifest. t.Run("add", func(t *testing.T) { @@ -113,14 +123,14 @@ func testManifestChange(t *testing.T, encrypt bool) { humansManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "up", humansDataFilename, ) newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "add", origManifestHash, @@ -168,14 +178,14 @@ func testManifestChange(t *testing.T, encrypt bool) { robotsManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "up", robotsDataFilename, ) newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "add", origManifestHash, @@ -228,14 +238,14 @@ func testManifestChange(t *testing.T, encrypt bool) { indexManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "up", indexDataFilename, ) newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "update", origManifestHash, @@ -286,14 +296,14 @@ func testManifestChange(t *testing.T, encrypt bool) { humansManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "up", robotsDataFilename, ) newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "update", origManifestHash, @@ -339,7 +349,7 @@ func testManifestChange(t *testing.T, encrypt bool) { t.Run("remove", func(t *testing.T) { newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "remove", origManifestHash, @@ -367,7 +377,7 @@ func testManifestChange(t *testing.T, encrypt bool) { t.Run("remove nested", func(t *testing.T) { newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "remove", origManifestHash, @@ -400,6 +410,10 @@ func testManifestChange(t *testing.T, encrypt bool) { // TestNestedDefaultEntryUpdate tests if the default entry is updated // if the file in nested manifest used for it is also updated. func TestNestedDefaultEntryUpdate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testNestedDefaultEntryUpdate(t, false) } @@ -407,13 +421,17 @@ func TestNestedDefaultEntryUpdate(t *testing.T) { // of encrypted upload is updated if the file in nested manifest // used for it is also updated. func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testNestedDefaultEntryUpdate(t, true) } func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { t.Parallel() - cluster := newTestCluster(t, 1) - defer cluster.Shutdown() + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) + defer srv.Close() tmp, err := ioutil.TempDir("", "swarm-manifest-test") if err != nil { @@ -441,7 +459,7 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { args := []string{ "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "--recursive", "--defaultpath", indexDataFilename, @@ -456,7 +474,7 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { checkHashLength(t, origManifestHash, encrypt) - client := swarm.NewClient(cluster.Nodes[0].URL) + client := swarm.NewClient(srv.URL) newIndexData := []byte("

Ethereum Swarm

") newIndexDataFilename := filepath.Join(tmp, "index.html") @@ -467,14 +485,14 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { newIndexManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "up", newIndexDataFilename, ) newManifestHash := runSwarmExpectHash(t, "--bzzapi", - cluster.Nodes[0].URL, + srv.URL, "manifest", "update", origManifestHash, diff --git a/cmd/swarm/mimegen/generator.go b/cmd/swarm/mimegen/generator.go new file mode 100644 index 0000000000..68f9e306e5 --- /dev/null +++ b/cmd/swarm/mimegen/generator.go @@ -0,0 +1,124 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package main + +// Standard "mime" package rely on system-settings, see mime.osInitMime +// Swarm will run on many OS/Platform/Docker and must behave similar +// This command generates code to add common mime types based on mime.types file +// +// mime.types file provided by mailcap, which follow https://www.iana.org/assignments/media-types/media-types.xhtml +// +// Get last version of mime.types file by: +// docker run --rm -v $(pwd):/tmp alpine:edge /bin/sh -c "apk add -U mailcap; mv /etc/mime.types /tmp" + +import ( + "bufio" + "bytes" + "flag" + "html/template" + "io/ioutil" + "strings" + + "log" +) + +var ( + typesFlag = flag.String("types", "", "Input mime.types file") + packageFlag = flag.String("package", "", "Golang package in output file") + outFlag = flag.String("out", "", "Output file name for the generated mime types") +) + +type mime struct { + Name string + Exts []string +} + +type templateParams struct { + PackageName string + Mimes []mime +} + +func main() { + // Parse and ensure all needed inputs are specified + flag.Parse() + if *typesFlag == "" { + log.Fatalf("--types is required") + } + if *packageFlag == "" { + log.Fatalf("--types is required") + } + if *outFlag == "" { + log.Fatalf("--out is required") + } + + params := templateParams{ + PackageName: *packageFlag, + } + + types, err := ioutil.ReadFile(*typesFlag) + if err != nil { + log.Fatal(err) + } + + scanner := bufio.NewScanner(bytes.NewReader(types)) + for scanner.Scan() { + txt := scanner.Text() + if strings.HasPrefix(txt, "#") || len(txt) == 0 { + continue + } + parts := strings.Fields(txt) + if len(parts) == 1 { + continue + } + params.Mimes = append(params.Mimes, mime{parts[0], parts[1:]}) + } + + if err = scanner.Err(); err != nil { + log.Fatal(err) + } + + result := bytes.NewBuffer([]byte{}) + + if err := template.Must(template.New("_").Parse(tpl)).Execute(result, params); err != nil { + log.Fatal(err) + } + + if err := ioutil.WriteFile(*outFlag, result.Bytes(), 0600); err != nil { + log.Fatal(err) + } +} + +var tpl = `// Code generated by github.com/ethereum/go-ethereum/cmd/swarm/mimegen. DO NOT EDIT. + +package {{ .PackageName }} + +import "mime" +func init() { + var mimeTypes = map[string]string{ +{{- range .Mimes -}} + {{ $name := .Name -}} + {{- range .Exts }} + ".{{ . }}": "{{ $name | html }}", + {{- end }} +{{- end }} + } + for ext, name := range mimeTypes { + if err := mime.AddExtensionType(ext, name); err != nil { + panic(err) + } + } +} +` diff --git a/cmd/swarm/mimegen/mime.types b/cmd/swarm/mimegen/mime.types new file mode 100644 index 0000000000..1bdf211490 --- /dev/null +++ b/cmd/swarm/mimegen/mime.types @@ -0,0 +1,1828 @@ +# This is a comment. I love comments. -*- indent-tabs-mode: t -*- + +# This file controls what Internet media types are sent to the client for +# given file extension(s). Sending the correct media type to the client +# is important so they know how to handle the content of the file. +# Extra types can either be added here or by using an AddType directive +# in your config files. For more information about Internet media types, +# please read RFC 2045, 2046, 2047, 2048, and 2077. The Internet media type +# registry is at . + +# IANA types + +# MIME type Extensions +application/1d-interleaved-parityfec +application/3gpdash-qoe-report+xml +application/3gpp-ims+xml +application/A2L a2l +application/activemessage +application/alto-costmap+json +application/alto-costmapfilter+json +application/alto-directory+json +application/alto-endpointcost+json +application/alto-endpointcostparams+json +application/alto-endpointprop+json +application/alto-endpointpropparams+json +application/alto-error+json +application/alto-networkmap+json +application/alto-networkmapfilter+json +application/AML aml +application/andrew-inset ez +application/applefile +application/ATF atf +application/ATFX atfx +application/ATXML atxml +application/atom+xml atom +application/atomcat+xml atomcat +application/atomdeleted+xml atomdeleted +application/atomicmail +application/atomsvc+xml atomsvc +application/auth-policy+xml apxml +application/bacnet-xdd+zip xdd +application/batch-SMTP +application/beep+xml +application/calendar+json +application/calendar+xml xcs +application/call-completion +application/cals-1840 +application/cbor cbor +application/ccmp+xml ccmp +application/ccxml+xml ccxml +application/CDFX+XML cdfx +application/cdmi-capability cdmia +application/cdmi-container cdmic +application/cdmi-domain cdmid +application/cdmi-object cdmio +application/cdmi-queue cdmiq +application/cdni +application/CEA cea +application/cea-2018+xml +application/cellml+xml cellml cml +application/cfw +application/clue_info+xml clue +application/cms cmsc +application/cnrp+xml +application/coap-group+json +application/coap-payload +application/commonground +application/conference-info+xml +application/cpl+xml cpl +application/cose +application/cose-key +application/cose-key-set +application/csrattrs csrattrs +application/csta+xml +application/CSTAdata+xml +application/csvm+json +application/cybercash +application/dash+xml mpd +application/dashdelta mpdd +application/davmount+xml davmount +application/dca-rft +application/DCD dcd +application/dec-dx +application/dialog-info+xml +application/dicom dcm +application/dicom+json +application/dicom+xml +application/DII dii +application/DIT dit +application/dns +application/dskpp+xml xmls +application/dssc+der dssc +application/dssc+xml xdssc +application/dvcs dvc +application/ecmascript es +application/EDI-Consent +application/EDI-X12 +application/EDIFACT +application/efi efi +application/EmergencyCallData.Comment+xml +application/EmergencyCallData.Control+xml +application/EmergencyCallData.DeviceInfo+xml +application/EmergencyCallData.eCall.MSD +application/EmergencyCallData.ProviderInfo+xml +application/EmergencyCallData.ServiceInfo+xml +application/EmergencyCallData.SubscriberInfo+xml +application/EmergencyCallData.VEDS+xml +application/emma+xml emma +application/emotionml+xml emotionml +application/encaprtp +application/epp+xml +application/epub+zip epub +application/eshop +application/exi exi +application/fastinfoset finf +application/fastsoap +application/fdt+xml fdt +# fits, fit, fts: image/fits +application/fits +# application/font-sfnt deprecated in favor of font/sfnt +application/font-tdpfr pfr +# application/font-woff deprecated in favor of font/woff +application/framework-attributes+xml +application/geo+json geojson +application/geo+json-seq +application/gml+xml gml +application/gzip gz tgz +application/H224 +application/held+xml +application/http +application/hyperstudio stk +application/ibe-key-request+xml +application/ibe-pkg-reply+xml +application/ibe-pp-data +application/iges +application/im-iscomposing+xml +application/index +application/index.cmd +application/index.obj +application/index.response +application/index.vnd +application/inkml+xml ink inkml +application/iotp +application/ipfix ipfix +application/ipp +application/isup +application/its+xml its +application/javascript js +application/jose +application/jose+json +application/jrd+json jrd +application/json json +application/json-patch+json json-patch +application/json-seq +application/jwk+json +application/jwk-set+json +application/jwt +application/kpml-request+xml +application/kpml-response+xml +application/ld+json jsonld +application/lgr+xml lgr +application/link-format wlnk +application/load-control+xml +application/lost+xml lostxml +application/lostsync+xml lostsyncxml +application/LXF lxf +application/mac-binhex40 hqx +application/macwriteii +application/mads+xml mads +application/marc mrc +application/marcxml+xml mrcx +application/mathematica nb ma mb +application/mathml-content+xml +application/mathml-presentation+xml +application/mathml+xml mml +application/mbms-associated-procedure-description+xml +application/mbms-deregister+xml +application/mbms-envelope+xml +application/mbms-msk-response+xml +application/mbms-msk+xml +application/mbms-protection-description+xml +application/mbms-reception-report+xml +application/mbms-register-response+xml +application/mbms-register+xml +application/mbms-schedule+xml +application/mbms-user-service-description+xml +application/mbox mbox +application/media_control+xml +# mpf: text/vnd.ms-mediapackage +application/media-policy-dataset+xml +application/mediaservercontrol+xml +application/merge-patch+json +application/metalink4+xml meta4 +application/mets+xml mets +application/MF4 mf4 +application/mikey +application/mods+xml mods +application/moss-keys +application/moss-signature +application/mosskey-data +application/mosskey-request +application/mp21 m21 mp21 +# mp4, mpg4: video/mp4, see RFC 4337 +application/mp4 +application/mpeg4-generic +application/mpeg4-iod +application/mpeg4-iod-xmt +# xdf: application/xcap-diff+xml +application/mrb-consumer+xml +application/mrb-publish+xml +application/msc-ivr+xml +application/msc-mixer+xml +application/msword doc +application/mud+json +application/mxf mxf +application/n-quads nq +application/n-triples nt +application/nasdata +application/news-checkgroups +application/news-groupinfo +application/news-transmission +application/nlsml+xml +application/nss +application/ocsp-request orq +application/ocsp-response ors +application/octet-stream bin lha lzh exe class so dll img iso +application/oda oda +application/ODX odx +application/oebps-package+xml opf +application/ogg ogx +application/oxps oxps +application/p2p-overlay+xml relo +application/parityfec +# xer: application/xcap-error+xml +application/patch-ops-error+xml +application/pdf pdf +application/PDX pdx +application/pgp-encrypted pgp +application/pgp-keys +application/pgp-signature sig +application/pidf-diff+xml +application/pidf+xml +application/pkcs10 p10 +application/pkcs12 p12 pfx +application/pkcs7-mime p7m p7c +application/pkcs7-signature p7s +application/pkcs8 p8 +# ac: application/vnd.nokia.n-gage.ac+xml +application/pkix-attr-cert +application/pkix-cert cer +application/pkix-crl crl +application/pkix-pkipath pkipath +application/pkixcmp pki +application/pls+xml pls +application/poc-settings+xml +application/postscript ps eps ai +application/ppsp-tracker+json +application/problem+json +application/problem+xml +application/provenance+xml provx +application/prs.alvestrand.titrax-sheet +application/prs.cww cw cww +application/prs.hpub+zip hpub +application/prs.nprend rnd rct +application/prs.plucker +application/prs.rdf-xml-crypt rdf-crypt +application/prs.xsf+xml xsf +application/pskc+xml pskcxml +application/qsig +application/raptorfec +application/rdap+json +application/rdf+xml rdf +application/reginfo+xml rif +application/relax-ng-compact-syntax rnc +application/remote-printing +application/reputon+json +application/resource-lists-diff+xml rld +application/resource-lists+xml rl +application/rfc+xml rfcxml +application/riscos +application/rlmi+xml +application/rls-services+xml rs +application/rpki-ghostbusters gbr +application/rpki-manifest mft +application/rpki-publication +application/rpki-roa roa +application/rpki-updown +application/rtf rtf +application/rtploopback +application/rtx +application/samlassertion+xml +application/samlmetadata+xml +application/sbml+xml +application/scaip+xml +# scm: application/vnd.lotus-screencam +application/scim+json scim +application/scvp-cv-request scq +application/scvp-cv-response scs +application/scvp-vp-request spq +application/scvp-vp-response spp +application/sdp sdp +application/sep+xml +application/sep-exi +application/session-info +application/set-payment +application/set-payment-initiation +application/set-registration +application/set-registration-initiation +application/sgml +application/sgml-open-catalog soc +application/shf+xml shf +application/sieve siv sieve +application/simple-filter+xml cl +application/simple-message-summary +application/simpleSymbolContainer +application/slate +# application/smil obsoleted by application/smil+xml +application/smil+xml smil smi sml +application/smpte336m +application/soap+fastinfoset +application/soap+xml +application/sparql-query rq +application/sparql-results+xml srx +application/spirits-event+xml +application/sql sql +application/srgs gram +application/srgs+xml grxml +application/sru+xml sru +application/ssml+xml ssml +application/tamp-apex-update tau +application/tamp-apex-update-confirm auc +application/tamp-community-update tcu +application/tamp-community-update-confirm cuc +application/tamp-error ter +application/tamp-sequence-adjust tsa +application/tamp-sequence-adjust-confirm sac +# tsq: application/timestamp-query +application/tamp-status-query +# tsr: application/timestamp-reply +application/tamp-status-response +application/tamp-update tur +application/tamp-update-confirm tuc +application/tei+xml tei teiCorpus odd +application/thraud+xml tfi +application/timestamp-query tsq +application/timestamp-reply tsr +application/timestamped-data tsd +application/trig trig +application/ttml+xml ttml +application/tve-trigger +application/ulpfec +application/urc-grpsheet+xml gsheet +application/urc-ressheet+xml rsheet +application/urc-targetdesc+xml td +application/urc-uisocketdesc+xml uis +application/vcard+json +application/vcard+xml +application/vemmi +application/vnd.3gpp.access-transfer-events+xml +application/vnd.3gpp.bsf+xml +application/vnd.3gpp.mid-call+xml +application/vnd.3gpp.pic-bw-large plb +application/vnd.3gpp.pic-bw-small psb +application/vnd.3gpp.pic-bw-var pvb +application/vnd.3gpp-prose+xml +application/vnd.3gpp-prose-pc3ch+xml +# sms: application/vnd.3gpp2.sms +application/vnd.3gpp.sms +application/vnd.3gpp.sms+xml +application/vnd.3gpp.srvcc-ext+xml +application/vnd.3gpp.SRVCC-info+xml +application/vnd.3gpp.state-and-event-info+xml +application/vnd.3gpp.ussd+xml +application/vnd.3gpp2.bcmcsinfo+xml +application/vnd.3gpp2.sms sms +application/vnd.3gpp2.tcap tcap +application/vnd.3lightssoftware.imagescal imgcal +application/vnd.3M.Post-it-Notes pwn +application/vnd.accpac.simply.aso aso +application/vnd.accpac.simply.imp imp +application/vnd.acucobol acu +application/vnd.acucorp atc acutc +application/vnd.adobe.flash.movie swf +application/vnd.adobe.formscentral.fcdt fcdt +application/vnd.adobe.fxp fxp fxpl +application/vnd.adobe.partial-upload +application/vnd.adobe.xdp+xml xdp +application/vnd.adobe.xfdf xfdf +application/vnd.aether.imp +application/vnd.ah-barcode +application/vnd.ahead.space ahead +application/vnd.airzip.filesecure.azf azf +application/vnd.airzip.filesecure.azs azs +application/vnd.amazon.mobi8-ebook azw3 +application/vnd.americandynamics.acc acc +application/vnd.amiga.ami ami +application/vnd.amundsen.maze+xml +application/vnd.anki apkg +application/vnd.anser-web-certificate-issue-initiation cii +# Not in IANA listing, but is on FTP site? +application/vnd.anser-web-funds-transfer-initiation fti +# atx: audio/ATRAC-X +application/vnd.antix.game-component +application/vnd.apache.thrift.binary +application/vnd.apache.thrift.compact +application/vnd.apache.thrift.json +application/vnd.api+json +application/vnd.apothekende.reservation+json +application/vnd.apple.installer+xml dist distz pkg mpkg +# m3u: audio/x-mpegurl for now +application/vnd.apple.mpegurl m3u8 +# application/vnd.arastra.swi obsoleted by application/vnd.aristanetworks.swi +application/vnd.aristanetworks.swi swi +application/vnd.artsquare +application/vnd.astraea-software.iota iota +application/vnd.audiograph aep +application/vnd.autopackage package +application/vnd.avistar+xml +application/vnd.balsamiq.bmml+xml bmml +application/vnd.balsamiq.bmpr bmpr +application/vnd.bekitzur-stech+json +application/vnd.bint.med-content +application/vnd.biopax.rdf+xml +application/vnd.blueice.multipass mpm +application/vnd.bluetooth.ep.oob ep +application/vnd.bluetooth.le.oob le +application/vnd.bmi bmi +application/vnd.businessobjects rep +application/vnd.cab-jscript +application/vnd.canon-cpdl +application/vnd.canon-lips +application/vnd.capasystems-pg+json +application/vnd.cendio.thinlinc.clientconf tlclient +application/vnd.century-systems.tcp_stream +application/vnd.chemdraw+xml cdxml +application/vnd.chess-pgn pgn +application/vnd.chipnuts.karaoke-mmd mmd +application/vnd.cinderella cdy +application/vnd.cirpack.isdn-ext +application/vnd.citationstyles.style+xml csl +application/vnd.claymore cla +application/vnd.cloanto.rp9 rp9 +application/vnd.clonk.c4group c4g c4d c4f c4p c4u +application/vnd.cluetrust.cartomobile-config c11amc +application/vnd.cluetrust.cartomobile-config-pkg c11amz +application/vnd.coffeescript coffee +application/vnd.collection+json +application/vnd.collection.doc+json +application/vnd.collection.next+json +application/vnd.comicbook+zip cbz +# icc: application/vnd.iccprofile +application/vnd.commerce-battelle ica icf icd ic0 ic1 ic2 ic3 ic4 ic5 ic6 ic7 ic8 +application/vnd.commonspace csp cst +application/vnd.contact.cmsg cdbcmsg +application/vnd.coreos.ignition+json ign ignition +application/vnd.cosmocaller cmc +application/vnd.crick.clicker clkx +application/vnd.crick.clicker.keyboard clkk +application/vnd.crick.clicker.palette clkp +application/vnd.crick.clicker.template clkt +application/vnd.crick.clicker.wordbank clkw +application/vnd.criticaltools.wbs+xml wbs +application/vnd.ctc-posml pml +application/vnd.ctct.ws+xml +application/vnd.cups-pdf +application/vnd.cups-postscript +application/vnd.cups-ppd ppd +application/vnd.cups-raster +application/vnd.cups-raw +application/vnd.curl curl +application/vnd.cyan.dean.root+xml +application/vnd.cybank +application/vnd.d2l.coursepackage1p0+zip +application/vnd.dart dart +application/vnd.data-vision.rdz rdz +application/vnd.datapackage+json +application/vnd.dataresource+json +application/vnd.debian.binary-package deb udeb +application/vnd.dece.data uvf uvvf uvd uvvd +application/vnd.dece.ttml+xml uvt uvvt +application/vnd.dece.unspecified uvx uvvx +application/vnd.dece.zip uvz uvvz +application/vnd.denovo.fcselayout-link fe_launch +application/vnd.desmume.movie dsm +application/vnd.dir-bi.plate-dl-nosuffix +application/vnd.dm.delegation+xml +application/vnd.dna dna +application/vnd.document+json docjson +application/vnd.dolby.mobile.1 +application/vnd.dolby.mobile.2 +application/vnd.doremir.scorecloud-binary-document scld +application/vnd.dpgraph dpg mwc dpgraph +application/vnd.dreamfactory dfac +application/vnd.drive+json +application/vnd.dtg.local +application/vnd.dtg.local.flash fla +application/vnd.dtg.local.html +application/vnd.dvb.ait ait +# class: application/octet-stream +application/vnd.dvb.dvbj +application/vnd.dvb.esgcontainer +application/vnd.dvb.ipdcdftnotifaccess +application/vnd.dvb.ipdcesgaccess +application/vnd.dvb.ipdcesgaccess2 +application/vnd.dvb.ipdcesgpdd +application/vnd.dvb.ipdcroaming +application/vnd.dvb.iptv.alfec-base +application/vnd.dvb.iptv.alfec-enhancement +application/vnd.dvb.notif-aggregate-root+xml +application/vnd.dvb.notif-container+xml +application/vnd.dvb.notif-generic+xml +application/vnd.dvb.notif-ia-msglist+xml +application/vnd.dvb.notif-ia-registration-request+xml +application/vnd.dvb.notif-ia-registration-response+xml +application/vnd.dvb.notif-init+xml +# pfr: application/font-tdpfr +application/vnd.dvb.pfr +application/vnd.dvb.service svc +# dxr: application/x-director +application/vnd.dxr +application/vnd.dynageo geo +application/vnd.dzr dzr +application/vnd.easykaraoke.cdgdownload +application/vnd.ecdis-update +application/vnd.ecowin.chart mag +application/vnd.ecowin.filerequest +application/vnd.ecowin.fileupdate +application/vnd.ecowin.series +application/vnd.ecowin.seriesrequest +application/vnd.ecowin.seriesupdate +# img: application/octet-stream +application/vnd.efi-img +# iso: application/octet-stream +application/vnd.efi-iso +application/vnd.enliven nml +application/vnd.enphase.envoy +application/vnd.eprints.data+xml +application/vnd.epson.esf esf +application/vnd.epson.msf msf +application/vnd.epson.quickanime qam +application/vnd.epson.salt slt +application/vnd.epson.ssf ssf +application/vnd.ericsson.quickcall qcall qca +application/vnd.espass-espass+zip espass +application/vnd.eszigno3+xml es3 et3 +application/vnd.etsi.aoc+xml +application/vnd.etsi.asic-e+zip asice sce +# scs: application/scvp-cv-response +application/vnd.etsi.asic-s+zip asics +application/vnd.etsi.cug+xml +application/vnd.etsi.iptvcommand+xml +application/vnd.etsi.iptvdiscovery+xml +application/vnd.etsi.iptvprofile+xml +application/vnd.etsi.iptvsad-bc+xml +application/vnd.etsi.iptvsad-cod+xml +application/vnd.etsi.iptvsad-npvr+xml +application/vnd.etsi.iptvservice+xml +application/vnd.etsi.iptvsync+xml +application/vnd.etsi.iptvueprofile+xml +application/vnd.etsi.mcid+xml +application/vnd.etsi.mheg5 +application/vnd.etsi.overload-control-policy-dataset+xml +application/vnd.etsi.pstn+xml +application/vnd.etsi.sci+xml +application/vnd.etsi.simservs+xml +application/vnd.etsi.timestamp-token tst +application/vnd.etsi.tsl.der +application/vnd.etsi.tsl+xml +application/vnd.eudora.data +application/vnd.ezpix-album ez2 +application/vnd.ezpix-package ez3 +application/vnd.f-secure.mobile +application/vnd.fastcopy-disk-image dim +application/vnd.fdf fdf +application/vnd.fdsn.mseed msd mseed +application/vnd.fdsn.seed seed dataless +application/vnd.ffsns +application/vnd.filmit.zfc zfc +# all extensions: application/vnd.hbci +application/vnd.fints +application/vnd.firemonkeys.cloudcell +application/vnd.FloGraphIt gph +application/vnd.fluxtime.clip ftc +application/vnd.font-fontforge-sfd sfd +application/vnd.framemaker fm +application/vnd.frogans.fnc fnc +application/vnd.frogans.ltf ltf +application/vnd.fsc.weblaunch fsc +application/vnd.fujitsu.oasys oas +application/vnd.fujitsu.oasys2 oa2 +application/vnd.fujitsu.oasys3 oa3 +application/vnd.fujitsu.oasysgp fg5 +application/vnd.fujitsu.oasysprs bh2 +application/vnd.fujixerox.ART-EX +application/vnd.fujixerox.ART4 +application/vnd.fujixerox.ddd ddd +application/vnd.fujixerox.docuworks xdw +application/vnd.fujixerox.docuworks.binder xbd +application/vnd.fujixerox.docuworks.container xct +application/vnd.fujixerox.HBPL +application/vnd.fut-misnet +application/vnd.fuzzysheet fzs +application/vnd.genomatix.tuxedo txd +# application/vnd.geo+json obsoleted by application/geo+json +application/vnd.geocube+xml g3 g³ +application/vnd.geogebra.file ggb +application/vnd.geogebra.tool ggt +application/vnd.geometry-explorer gex gre +application/vnd.geonext gxt +application/vnd.geoplan g2w +application/vnd.geospace g3w +# gbr: application/rpki-ghostbusters +application/vnd.gerber +application/vnd.globalplatform.card-content-mgt +application/vnd.globalplatform.card-content-mgt-response +application/vnd.gmx gmx +application/vnd.google-earth.kml+xml kml +application/vnd.google-earth.kmz kmz +application/vnd.gov.sk.e-form+xml +application/vnd.gov.sk.e-form+zip +application/vnd.gov.sk.xmldatacontainer+xml +application/vnd.grafeq gqf gqs +application/vnd.gridmp +application/vnd.groove-account gac +application/vnd.groove-help ghf +application/vnd.groove-identity-message gim +application/vnd.groove-injector grv +application/vnd.groove-tool-message gtm +application/vnd.groove-tool-template tpl +application/vnd.groove-vcard vcg +application/vnd.hal+json +application/vnd.hal+xml hal +application/vnd.HandHeld-Entertainment+xml zmm +application/vnd.hbci hbci hbc kom upa pkd bpd +application/vnd.hc+json +# rep: application/vnd.businessobjects +application/vnd.hcl-bireports +application/vnd.hdt hdt +application/vnd.heroku+json +application/vnd.hhe.lesson-player les +application/vnd.hp-HPGL hpgl +application/vnd.hp-hpid hpi hpid +application/vnd.hp-hps hps +application/vnd.hp-jlyt jlt +application/vnd.hp-PCL pcl +application/vnd.hp-PCLXL +application/vnd.httphone +application/vnd.hydrostatix.sof-data sfd-hdstx +application/vnd.hyperdrive+json +application/vnd.hzn-3d-crossword x3d +application/vnd.ibm.afplinedata +application/vnd.ibm.electronic-media emm +application/vnd.ibm.MiniPay mpy +application/vnd.ibm.modcap list3820 listafp afp pseg3820 +application/vnd.ibm.rights-management irm +application/vnd.ibm.secure-container sc +application/vnd.iccprofile icc icm +application/vnd.ieee.1905 1905.1 +application/vnd.igloader igl +application/vnd.imagemeter.folder+zip imf +application/vnd.imagemeter.image+zip imi +application/vnd.immervision-ivp ivp +application/vnd.immervision-ivu ivu +application/vnd.ims.imsccv1p1 imscc +application/vnd.ims.imsccv1p2 +application/vnd.ims.imsccv1p3 +application/vnd.ims.lis.v2.result+json +application/vnd.ims.lti.v2.toolconsumerprofile+json +application/vnd.ims.lti.v2.toolproxy.id+json +application/vnd.ims.lti.v2.toolproxy+json +application/vnd.ims.lti.v2.toolsettings+json +application/vnd.ims.lti.v2.toolsettings.simple+json +application/vnd.informedcontrol.rms+xml +# application/vnd.informix-visionary obsoleted by application/vnd.visionary +application/vnd.infotech.project +application/vnd.infotech.project+xml +application/vnd.innopath.wamp.notification +application/vnd.insors.igm igm +application/vnd.intercon.formnet xpw xpx +application/vnd.intergeo i2g +application/vnd.intertrust.digibox +application/vnd.intertrust.nncp +application/vnd.intu.qbo qbo +application/vnd.intu.qfx qfx +application/vnd.iptc.g2.catalogitem+xml +application/vnd.iptc.g2.conceptitem+xml +application/vnd.iptc.g2.knowledgeitem+xml +application/vnd.iptc.g2.newsitem+xml +application/vnd.iptc.g2.newsmessage+xml +application/vnd.iptc.g2.packageitem+xml +application/vnd.iptc.g2.planningitem+xml +application/vnd.ipunplugged.rcprofile rcprofile +application/vnd.irepository.package+xml irp +application/vnd.is-xpr xpr +application/vnd.isac.fcs fcs +application/vnd.jam jam +application/vnd.japannet-directory-service +application/vnd.japannet-jpnstore-wakeup +application/vnd.japannet-payment-wakeup +application/vnd.japannet-registration +application/vnd.japannet-registration-wakeup +application/vnd.japannet-setstore-wakeup +application/vnd.japannet-verification +application/vnd.japannet-verification-wakeup +application/vnd.jcp.javame.midlet-rms rms +application/vnd.jisp jisp +application/vnd.joost.joda-archive joda +application/vnd.jsk.isdn-ngn +application/vnd.kahootz ktz ktr +application/vnd.kde.karbon karbon +application/vnd.kde.kchart chrt +application/vnd.kde.kformula kfo +application/vnd.kde.kivio flw +application/vnd.kde.kontour kon +application/vnd.kde.kpresenter kpr kpt +application/vnd.kde.kspread ksp +application/vnd.kde.kword kwd kwt +application/vnd.kenameaapp htke +application/vnd.kidspiration kia +application/vnd.Kinar kne knp sdf +application/vnd.koan skp skd skm skt +application/vnd.kodak-descriptor sse +application/vnd.las.las+json lasjson +application/vnd.las.las+xml lasxml +application/vnd.liberty-request+xml +application/vnd.llamagraphics.life-balance.desktop lbd +application/vnd.llamagraphics.life-balance.exchange+xml lbe +application/vnd.lotus-1-2-3 123 wk4 wk3 wk1 +application/vnd.lotus-approach apr vew +application/vnd.lotus-freelance prz pre +application/vnd.lotus-notes nsf ntf ndl ns4 ns3 ns2 nsh nsg +application/vnd.lotus-organizer or3 or2 org +application/vnd.lotus-screencam scm +application/vnd.lotus-wordpro lwp sam +application/vnd.macports.portpkg portpkg +application/vnd.mapbox-vector-tile mvt +application/vnd.marlin.drm.actiontoken+xml +application/vnd.marlin.drm.conftoken+xml +application/vnd.marlin.drm.license+xml +application/vnd.marlin.drm.mdcf mdc +application/vnd.mason+json +application/vnd.maxmind.maxmind-db mmdb +application/vnd.mcd mcd +application/vnd.medcalcdata mc1 +application/vnd.mediastation.cdkey cdkey +application/vnd.meridian-slingshot +application/vnd.MFER mwf +application/vnd.mfmp mfm +application/vnd.micro+json +application/vnd.micrografx.flo flo +application/vnd.micrografx.igx igx +application/vnd.microsoft.portable-executable +application/vnd.microsoft.windows.thumbnail-cache +application/vnd.miele+json +application/vnd.mif mif +application/vnd.minisoft-hp3000-save +application/vnd.mitsubishi.misty-guard.trustweb +application/vnd.Mobius.DAF daf +application/vnd.Mobius.DIS dis +application/vnd.Mobius.MBK mbk +application/vnd.Mobius.MQY mqy +application/vnd.Mobius.MSL msl +application/vnd.Mobius.PLC plc +application/vnd.Mobius.TXF txf +application/vnd.mophun.application mpn +application/vnd.mophun.certificate mpc +application/vnd.motorola.flexsuite +application/vnd.motorola.flexsuite.adsi +application/vnd.motorola.flexsuite.fis +application/vnd.motorola.flexsuite.gotap +application/vnd.motorola.flexsuite.kmr +application/vnd.motorola.flexsuite.ttc +application/vnd.motorola.flexsuite.wem +application/vnd.motorola.iprm +application/vnd.mozilla.xul+xml xul +application/vnd.ms-3mfdocument 3mf +application/vnd.ms-artgalry cil +application/vnd.ms-asf asf +application/vnd.ms-cab-compressed cab +application/vnd.ms-excel xls xlm xla xlc xlt xlw +application/vnd.ms-excel.template.macroEnabled.12 xltm +application/vnd.ms-excel.addin.macroEnabled.12 xlam +application/vnd.ms-excel.sheet.binary.macroEnabled.12 xlsb +application/vnd.ms-excel.sheet.macroEnabled.12 xlsm +application/vnd.ms-fontobject eot +application/vnd.ms-htmlhelp chm +application/vnd.ms-ims ims +application/vnd.ms-lrm lrm +application/vnd.ms-office.activeX+xml +application/vnd.ms-officetheme thmx +application/vnd.ms-playready.initiator+xml +application/vnd.ms-powerpoint ppt pps pot +application/vnd.ms-powerpoint.addin.macroEnabled.12 ppam +application/vnd.ms-powerpoint.presentation.macroEnabled.12 pptm +application/vnd.ms-powerpoint.slide.macroEnabled.12 sldm +application/vnd.ms-powerpoint.slideshow.macroEnabled.12 ppsm +application/vnd.ms-powerpoint.template.macroEnabled.12 potm +application/vnd.ms-PrintDeviceCapabilities+xml +application/vnd.ms-PrintSchemaTicket+xml +application/vnd.ms-project mpp mpt +application/vnd.ms-tnef tnef tnf +application/vnd.ms-windows.devicepairing +application/vnd.ms-windows.nwprinting.oob +application/vnd.ms-windows.printerpairing +application/vnd.ms-windows.wsd.oob +application/vnd.ms-wmdrm.lic-chlg-req +application/vnd.ms-wmdrm.lic-resp +application/vnd.ms-wmdrm.meter-chlg-req +application/vnd.ms-wmdrm.meter-resp +application/vnd.ms-word.document.macroEnabled.12 docm +application/vnd.ms-word.template.macroEnabled.12 dotm +application/vnd.ms-works wcm wdb wks wps +application/vnd.ms-wpl wpl +application/vnd.ms-xpsdocument xps +application/vnd.msa-disk-image msa +application/vnd.mseq mseq +application/vnd.msign +application/vnd.multiad.creator crtr +application/vnd.multiad.creator.cif cif +application/vnd.music-niff +application/vnd.musician mus +application/vnd.muvee.style msty +application/vnd.mynfc taglet +application/vnd.ncd.control +application/vnd.ncd.reference +application/vnd.nearst.inv+json +application/vnd.nervana entity request bkm kcm +application/vnd.netfpx +# ntf: application/vnd.lotus-notes +application/vnd.nitf nitf +application/vnd.neurolanguage.nlu nlu +application/vnd.nintendo.nitro.rom nds +application/vnd.nintendo.snes.rom sfc smc +application/vnd.noblenet-directory nnd +application/vnd.noblenet-sealer nns +application/vnd.noblenet-web nnw +application/vnd.nokia.catalogs +application/vnd.nokia.conml+wbxml +application/vnd.nokia.conml+xml +application/vnd.nokia.iptv.config+xml +application/vnd.nokia.iSDS-radio-presets +application/vnd.nokia.landmark+wbxml +application/vnd.nokia.landmark+xml +application/vnd.nokia.landmarkcollection+xml +application/vnd.nokia.n-gage.ac+xml ac +application/vnd.nokia.n-gage.data ngdat +application/vnd.nokia.n-gage.symbian.install n-gage +application/vnd.nokia.ncd +application/vnd.nokia.pcd+wbxml +application/vnd.nokia.pcd+xml +application/vnd.nokia.radio-preset rpst +application/vnd.nokia.radio-presets rpss +application/vnd.novadigm.EDM edm +application/vnd.novadigm.EDX edx +application/vnd.novadigm.EXT ext +application/vnd.ntt-local.content-share +application/vnd.ntt-local.file-transfer +application/vnd.ntt-local.ogw_remote-access +application/vnd.ntt-local.sip-ta_remote +application/vnd.ntt-local.sip-ta_tcp_stream +application/vnd.oasis.opendocument.chart odc +application/vnd.oasis.opendocument.chart-template otc +application/vnd.oasis.opendocument.database odb +application/vnd.oasis.opendocument.formula odf +# otf: font/otf +application/vnd.oasis.opendocument.formula-template +application/vnd.oasis.opendocument.graphics odg +application/vnd.oasis.opendocument.graphics-template otg +application/vnd.oasis.opendocument.image odi +application/vnd.oasis.opendocument.image-template oti +application/vnd.oasis.opendocument.presentation odp +application/vnd.oasis.opendocument.presentation-template otp +application/vnd.oasis.opendocument.spreadsheet ods +application/vnd.oasis.opendocument.spreadsheet-template ots +application/vnd.oasis.opendocument.text odt +application/vnd.oasis.opendocument.text-master odm +application/vnd.oasis.opendocument.text-template ott +application/vnd.oasis.opendocument.text-web oth +application/vnd.obn +application/vnd.ocf+cbor +application/vnd.oftn.l10n+json +application/vnd.oipf.contentaccessdownload+xml +application/vnd.oipf.contentaccessstreaming+xml +application/vnd.oipf.cspg-hexbinary +application/vnd.oipf.dae.svg+xml +application/vnd.oipf.dae.xhtml+xml +application/vnd.oipf.mippvcontrolmessage+xml +application/vnd.oipf.pae.gem +application/vnd.oipf.spdiscovery+xml +application/vnd.oipf.spdlist+xml +application/vnd.oipf.ueprofile+xml +application/vnd.olpc-sugar xo +application/vnd.oma.bcast.associated-procedure-parameter+xml +application/vnd.oma.bcast.drm-trigger+xml +application/vnd.oma.bcast.imd+xml +application/vnd.oma.bcast.ltkm +application/vnd.oma.bcast.notification+xml +application/vnd.oma.bcast.provisioningtrigger +application/vnd.oma.bcast.sgboot +application/vnd.oma.bcast.sgdd+xml +application/vnd.oma.bcast.sgdu +application/vnd.oma.bcast.simple-symbol-container +application/vnd.oma.bcast.smartcard-trigger+xml +application/vnd.oma.bcast.sprov+xml +application/vnd.oma.bcast.stkm +application/vnd.oma.cab-address-book+xml +application/vnd.oma.cab-feature-handler+xml +application/vnd.oma.cab-pcc+xml +application/vnd.oma.cab-subs-invite+xml +application/vnd.oma.cab-user-prefs+xml +application/vnd.oma.dcd +application/vnd.oma.dcdc +application/vnd.oma.dd2+xml dd2 +application/vnd.oma.drm.risd+xml +application/vnd.oma.group-usage-list+xml +application/vnd.oma.lwm2m+json +application/vnd.oma.lwm2m+tlv +application/vnd.oma.pal+xml +application/vnd.oma.poc.detailed-progress-report+xml +application/vnd.oma.poc.final-report+xml +application/vnd.oma.poc.groups+xml +application/vnd.oma.poc.invocation-descriptor+xml +application/vnd.oma.poc.optimized-progress-report+xml +application/vnd.oma.push +application/vnd.oma.scidm.messages+xml +application/vnd.oma.xcap-directory+xml +application/vnd.oma-scws-config +application/vnd.oma-scws-http-request +application/vnd.oma-scws-http-response +application/vnd.omads-email+xml +application/vnd.omads-file+xml +application/vnd.omads-folder+xml +application/vnd.omaloc-supl-init +application/vnd.onepager tam +application/vnd.onepagertamp tamp +application/vnd.onepagertamx tamx +application/vnd.onepagertat tat +application/vnd.onepagertatp tatp +application/vnd.onepagertatx tatx +application/vnd.openblox.game+xml obgx +application/vnd.openblox.game-binary obg +application/vnd.openeye.oeb oeb +application/vnd.openofficeorg.extension oxt +application/vnd.openstreetmap.data+xml osm +application/vnd.openxmlformats-officedocument.custom-properties+xml +application/vnd.openxmlformats-officedocument.customXmlProperties+xml +application/vnd.openxmlformats-officedocument.drawing+xml +application/vnd.openxmlformats-officedocument.drawingml.chart+xml +application/vnd.openxmlformats-officedocument.drawingml.chartshapes+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramColors+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramData+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramLayout+xml +application/vnd.openxmlformats-officedocument.drawingml.diagramStyle+xml +application/vnd.openxmlformats-officedocument.extended-properties+xml +application/vnd.openxmlformats-officedocument.presentationml.commentAuthors+xml +application/vnd.openxmlformats-officedocument.presentationml.comments+xml +application/vnd.openxmlformats-officedocument.presentationml.handoutMaster+xml +application/vnd.openxmlformats-officedocument.presentationml.notesMaster+xml +application/vnd.openxmlformats-officedocument.presentationml.notesSlide+xml +application/vnd.openxmlformats-officedocument.presentationml.presProps+xml +application/vnd.openxmlformats-officedocument.presentationml.presentation pptx +application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml +application/vnd.openxmlformats-officedocument.presentationml.slide sldx +application/vnd.openxmlformats-officedocument.presentationml.slide+xml +application/vnd.openxmlformats-officedocument.presentationml.slideLayout+xml +application/vnd.openxmlformats-officedocument.presentationml.slideMaster+xml +application/vnd.openxmlformats-officedocument.presentationml.slideUpdateInfo+xml +application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx +application/vnd.openxmlformats-officedocument.presentationml.slideshow.main+xml +application/vnd.openxmlformats-officedocument.presentationml.tableStyles+xml +application/vnd.openxmlformats-officedocument.presentationml.tags+xml +application/vnd.openxmlformats-officedocument.presentationml.template potx +application/vnd.openxmlformats-officedocument.presentationml.template.main+xml +application/vnd.openxmlformats-officedocument.presentationml.viewProps+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.calcChain+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.connections+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.dialogsheet+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.externalLink+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.queryTable+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.revisionHeaders+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.revisionLog+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.sheetMetadata+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.table+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.tableSingleCells+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx +application/vnd.openxmlformats-officedocument.spreadsheetml.template.main+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.userNames+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.volatileDependencies+xml +application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml +application/vnd.openxmlformats-officedocument.theme+xml +application/vnd.openxmlformats-officedocument.themeOverride+xml +application/vnd.openxmlformats-officedocument.vmlDrawing +application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.document docx +application/vnd.openxmlformats-officedocument.wordprocessingml.document.glossary+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.endnotes+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.fontTable+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx +application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml +application/vnd.openxmlformats-officedocument.wordprocessingml.webSettings+xml +application/vnd.openxmlformats-package.core-properties+xml +application/vnd.openxmlformats-package.digital-signature-xmlsignature+xml +application/vnd.openxmlformats-package.relationships+xml +application/vnd.oracle.resource+json +application/vnd.orange.indata +application/vnd.osa.netdeploy ndc +application/vnd.osgeo.mapguide.package mgp +# jar: application/x-java-archive +application/vnd.osgi.bundle +application/vnd.osgi.dp dp +application/vnd.osgi.subsystem esa +application/vnd.otps.ct-kip+xml +application/vnd.oxli.countgraph oxlicg +application/vnd.pagerduty+json +application/vnd.palm prc pdb pqa oprc +application/vnd.panoply plp +application/vnd.paos+xml +application/vnd.pawaafile paw +application/vnd.pcos +application/vnd.pg.format str +application/vnd.pg.osasli ei6 +application/vnd.piaccess.application-license pil +application/vnd.picsel efif +application/vnd.pmi.widget wg +application/vnd.poc.group-advertisement+xml +application/vnd.pocketlearn plf +application/vnd.powerbuilder6 pbd +application/vnd.powerbuilder6-s +application/vnd.powerbuilder7 +application/vnd.powerbuilder7-s +application/vnd.powerbuilder75 +application/vnd.powerbuilder75-s +application/vnd.preminet preminet +application/vnd.previewsystems.box box vbox +application/vnd.proteus.magazine mgz +application/vnd.publishare-delta-tree qps +# pti: image/prs.pti +application/vnd.pvi.ptid1 ptid +application/vnd.pwg-multiplexed +application/vnd.pwg-xhtml-print+xml +application/vnd.qualcomm.brew-app-res bar +application/vnd.quarantainenet +application/vnd.Quark.QuarkXPress qxd qxt qwd qwt qxl qxb +application/vnd.quobject-quoxdocument quox quiz +application/vnd.radisys.moml+xml +application/vnd.radisys.msml-audit-conf+xml +application/vnd.radisys.msml-audit-conn+xml +application/vnd.radisys.msml-audit-dialog+xml +application/vnd.radisys.msml-audit-stream+xml +application/vnd.radisys.msml-audit+xml +application/vnd.radisys.msml-conf+xml +application/vnd.radisys.msml-dialog-base+xml +application/vnd.radisys.msml-dialog-fax-detect+xml +application/vnd.radisys.msml-dialog-fax-sendrecv+xml +application/vnd.radisys.msml-dialog-group+xml +application/vnd.radisys.msml-dialog-speech+xml +application/vnd.radisys.msml-dialog-transform+xml +application/vnd.radisys.msml-dialog+xml +application/vnd.radisys.msml+xml +application/vnd.rainstor.data tree +application/vnd.rapid +application/vnd.rar rar +application/vnd.realvnc.bed bed +application/vnd.recordare.musicxml mxl +application/vnd.recordare.musicxml+xml +application/vnd.RenLearn.rlprint +application/vnd.rig.cryptonote cryptonote +application/vnd.route66.link66+xml link66 +# gbr: application/rpki-ghostbusters +application/vnd.rs-274x +application/vnd.ruckus.download +application/vnd.s3sms +application/vnd.sailingtracker.track st +application/vnd.sbm.cid +application/vnd.sbm.mid2 +application/vnd.scribus scd sla slaz +application/vnd.sealed.3df s3df +application/vnd.sealed.csf scsf +application/vnd.sealed.doc sdoc sdo s1w +application/vnd.sealed.eml seml sem +application/vnd.sealed.mht smht smh +application/vnd.sealed.net +# spp: application/scvp-vp-response +application/vnd.sealed.ppt sppt s1p +application/vnd.sealed.tiff stif +application/vnd.sealed.xls sxls sxl s1e +# stm: audio/x-stm +application/vnd.sealedmedia.softseal.html stml s1h +application/vnd.sealedmedia.softseal.pdf spdf spd s1a +application/vnd.seemail see +application/vnd.sema sema +application/vnd.semd semd +application/vnd.semf semf +application/vnd.shana.informed.formdata ifm +application/vnd.shana.informed.formtemplate itp +application/vnd.shana.informed.interchange iif +application/vnd.shana.informed.package ipk +application/vnd.SimTech-MindMapper twd twds +application/vnd.siren+json +application/vnd.smaf mmf +application/vnd.smart.notebook notebook +application/vnd.smart.teacher teacher +application/vnd.software602.filler.form+xml fo +application/vnd.software602.filler.form-xml-zip zfo +application/vnd.solent.sdkm+xml sdkm sdkd +application/vnd.spotfire.dxp dxp +application/vnd.spotfire.sfs sfs +application/vnd.sss-cod +application/vnd.sss-dtf +application/vnd.sss-ntf +application/vnd.stepmania.package smzip +application/vnd.stepmania.stepchart sm +application/vnd.street-stream +application/vnd.sun.wadl+xml wadl +application/vnd.sus-calendar sus susp +application/vnd.svd +application/vnd.swiftview-ics +application/vnd.syncml+xml xsm +application/vnd.syncml.dm+wbxml bdm +application/vnd.syncml.dm+xml xdm +application/vnd.syncml.dm.notification +application/vnd.syncml.dmddf+wbxml +application/vnd.syncml.dmddf+xml ddf +application/vnd.syncml.dmtnds+wbxml +application/vnd.syncml.dmtnds+xml +application/vnd.syncml.ds.notification +application/vnd.tableschema+json +application/vnd.tao.intent-module-archive tao +application/vnd.tcpdump.pcap pcap cap dmp +application/vnd.theqvd qvd +application/vnd.tmd.mediaflex.api+xml +application/vnd.tml vfr viaframe +application/vnd.tmobile-livetv tmo +application/vnd.tri.onesource +application/vnd.trid.tpt tpt +application/vnd.triscape.mxs mxs +application/vnd.trueapp tra +application/vnd.truedoc +# cab: application/vnd.ms-cab-compressed +application/vnd.ubisoft.webplayer +application/vnd.ufdl ufdl ufd frm +application/vnd.uiq.theme utz +application/vnd.umajin umj +application/vnd.unity unityweb +application/vnd.uoml+xml uoml uo +application/vnd.uplanet.alert +application/vnd.uplanet.alert-wbxml +application/vnd.uplanet.bearer-choice +application/vnd.uplanet.bearer-choice-wbxml +application/vnd.uplanet.cacheop +application/vnd.uplanet.cacheop-wbxml +application/vnd.uplanet.channel +application/vnd.uplanet.channel-wbxml +application/vnd.uplanet.list +application/vnd.uplanet.list-wbxml +application/vnd.uplanet.listcmd +application/vnd.uplanet.listcmd-wbxml +application/vnd.uplanet.signal +application/vnd.uri-map urim urimap +application/vnd.valve.source.material vmt +application/vnd.vcx vcx +# sxi: application/vnd.sun.xml.impress +application/vnd.vd-study mxi study-inter model-inter +# mcd: application/vnd.mcd +application/vnd.vectorworks vwx +application/vnd.vel+json +application/vnd.verimatrix.vcas +application/vnd.vidsoft.vidconference vsc +application/vnd.visio vsd vst vsw vss +application/vnd.visionary vis +# vsc: application/vnd.vidsoft.vidconference +application/vnd.vividence.scriptfile +application/vnd.vsf vsf +application/vnd.wap.sic sic +application/vnd.wap.slc slc +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo wtb +application/vnd.wfa.p2p p2p +application/vnd.wfa.wsc wsc +application/vnd.windows.devicepairing +application/vnd.wmc wmc +application/vnd.wmf.bootstrap +# nb: application/mathematica for now +application/vnd.wolfram.mathematica +application/vnd.wolfram.mathematica.package m +application/vnd.wolfram.player nbp +application/vnd.wordperfect wpd +application/vnd.wqd wqd +application/vnd.wrq-hp3000-labelled +application/vnd.wt.stf stf +application/vnd.wv.csp+xml +application/vnd.wv.csp+wbxml wv +application/vnd.wv.ssp+xml +application/vnd.xacml+json +application/vnd.xara xar +application/vnd.xfdl xfdl xfd +application/vnd.xfdl.webform +application/vnd.xmi+xml +application/vnd.xmpie.cpkg cpkg +application/vnd.xmpie.dpkg dpkg +# dpkg: application/vnd.xmpie.dpkg +application/vnd.xmpie.plan +application/vnd.xmpie.ppkg ppkg +application/vnd.xmpie.xlim xlim +application/vnd.yamaha.hv-dic hvd +application/vnd.yamaha.hv-script hvs +application/vnd.yamaha.hv-voice hvp +application/vnd.yamaha.openscoreformat osf +application/vnd.yamaha.openscoreformat.osfpvg+xml +application/vnd.yamaha.remote-setup +application/vnd.yamaha.smaf-audio saf +application/vnd.yamaha.smaf-phrase spf +application/vnd.yamaha.through-ngn +application/vnd.yamaha.tunnel-udpencap +application/vnd.yaoweme yme +application/vnd.yellowriver-custom-menu cmp +application/vnd.zul zir zirz +application/vnd.zzazz.deck+xml zaz +application/voicexml+xml vxml +application/vq-rtcp-xr +application/watcherinfo+xml wif +application/whoispp-query +application/whoispp-response +application/widget wgt +application/wita +application/wordperfect5.1 +application/wsdl+xml wsdl +application/wspolicy+xml wspolicy +# yes, this *is* IANA registered despite of x- +application/x-www-form-urlencoded +application/x400-bp +application/xacml+xml +application/xcap-att+xml xav +application/xcap-caps+xml xca +application/xcap-diff+xml xdf +application/xcap-el+xml xel +application/xcap-error+xml xer +application/xcap-ns+xml xns +application/xcon-conference-info-diff+xml +application/xcon-conference-info+xml +application/xenc+xml +application/xhtml+xml xhtml xhtm xht +# xml, xsd, rng: text/xml +application/xml +# mod: audio/x-mod +application/xml-dtd dtd +# ent: text/xml-external-parsed-entity +application/xml-external-parsed-entity +application/xml-patch+xml +application/xmpp+xml +application/xop+xml xop +application/xslt+xml xsl xslt +application/xv+xml mxml xhvml xvml xvm +application/yang yang +application/yang-data+json +application/yang-data+xml +application/yang-patch+json +application/yang-patch+xml +application/yin+xml yin +application/zip zip +application/zlib +audio/1d-interleaved-parityfec +audio/32kadpcm 726 +# 3gp, 3gpp: video/3gpp +audio/3gpp +# 3g2, 3gpp2: video/3gpp2 +audio/3gpp2 +audio/ac3 ac3 +audio/AMR amr +audio/AMR-WB awb +audio/amr-wb+ +audio/aptx +audio/asc acn +# aa3, omg: audio/ATRAC3 +audio/ATRAC-ADVANCED-LOSSLESS aal +# aa3, omg: audio/ATRAC3 +audio/ATRAC-X atx +audio/ATRAC3 at3 aa3 omg +audio/basic au snd +audio/BV16 +audio/BV32 +audio/clearmode +audio/CN +audio/DAT12 +audio/dls dls +audio/dsr-es201108 +audio/dsr-es202050 +audio/dsr-es202211 +audio/dsr-es202212 +audio/DV +audio/DVI4 +audio/eac3 +audio/encaprtp +audio/EVRC evc +# qcp: audio/qcelp +audio/EVRC-QCP +audio/EVRC0 +audio/EVRC1 +audio/EVRCB evb +audio/EVRCB0 +audio/EVRCB1 +audio/EVRCNW enw +audio/EVRCNW0 +audio/EVRCNW1 +audio/EVRCWB evw +audio/EVRCWB0 +audio/EVRCWB1 +audio/EVS +audio/example +audio/fwdred +audio/G711-0 +audio/G719 +audio/G722 +audio/G7221 +audio/G723 +audio/G726-16 +audio/G726-24 +audio/G726-32 +audio/G726-40 +audio/G728 +audio/G729 +audio/G7291 +audio/G729D +audio/G729E +audio/GSM +audio/GSM-EFR +audio/GSM-HR-08 +audio/iLBC lbc +audio/ip-mr_v2.5 +# wav: audio/x-wav +audio/L16 l16 +audio/L20 +audio/L24 +audio/L8 +audio/LPC +audio/MELP +audio/MELP600 +audio/MELP1200 +audio/MELP2400 +audio/mobile-xmf mxmf +# mp4, mpg4: video/mp4, see RFC 4337 +audio/mp4 m4a +audio/MP4A-LATM +audio/MPA +audio/mpa-robust +audio/mpeg mp3 mpga mp1 mp2 +audio/mpeg4-generic +audio/ogg oga ogg opus spx +audio/opus +audio/parityfec +audio/PCMA +audio/PCMA-WB +audio/PCMU +audio/PCMU-WB +audio/prs.sid sid psid +audio/qcelp qcp +audio/raptorfec +audio/RED +audio/rtp-enc-aescm128 +audio/rtp-midi +audio/rtploopback +audio/rtx +audio/SMV smv +# qcp: audio/qcelp, see RFC 3625 +audio/SMV-QCP +audio/SMV0 +# mid: audio/midi +audio/sp-midi +audio/speex +audio/t140c +audio/t38 +audio/telephone-event +audio/tone +audio/UEMCLIP +audio/ulpfec +audio/VDVI +audio/VMR-WB +audio/vnd.3gpp.iufp +audio/vnd.4SB +audio/vnd.audikoz koz +audio/vnd.CELP +audio/vnd.cisco.nse +audio/vnd.cmles.radio-events +audio/vnd.cns.anp1 +audio/vnd.cns.inf1 +audio/vnd.dece.audio uva uvva +audio/vnd.digital-winds eol +audio/vnd.dlna.adts +audio/vnd.dolby.heaac.1 +audio/vnd.dolby.heaac.2 +audio/vnd.dolby.mlp mlp +audio/vnd.dolby.mps +audio/vnd.dolby.pl2 +audio/vnd.dolby.pl2x +audio/vnd.dolby.pl2z +audio/vnd.dolby.pulse.1 +audio/vnd.dra +# wav: audio/x-wav, cpt: application/mac-compactpro +audio/vnd.dts dts +audio/vnd.dts.hd dtshd +# dvb: video/vnd.dvb.file +audio/vnd.dvb.file +audio/vnd.everad.plj plj +# rm: audio/x-pn-realaudio +audio/vnd.hns.audio +audio/vnd.lucent.voice lvp +audio/vnd.ms-playready.media.pya pya +# mxmf: audio/mobile-xmf +audio/vnd.nokia.mobile-xmf +audio/vnd.nortel.vbk vbk +audio/vnd.nuera.ecelp4800 ecelp4800 +audio/vnd.nuera.ecelp7470 ecelp7470 +audio/vnd.nuera.ecelp9600 ecelp9600 +audio/vnd.octel.sbc +# audio/vnd.qcelp deprecated in favour of audio/qcelp +audio/vnd.rhetorex.32kadpcm +audio/vnd.rip rip +audio/vnd.sealedmedia.softseal.mpeg smp3 smp s1m +audio/vnd.vmx.cvsd +audio/vorbis +audio/vorbis-config +font/collection ttc +font/otf otf +font/sfnt +font/ttf ttf +font/woff woff +font/woff2 woff2 +image/bmp bmp dib +image/cgm cgm +image/dicom-rle drle +image/emf emf +image/example +image/fits fits fit fts +image/g3fax +image/gif gif +image/ief ief +image/jls jls +image/jp2 jp2 jpg2 +image/jpeg jpg jpeg jpe jfif +image/jpm jpm jpgm +image/jpx jpx jpf +image/ktx ktx +image/naplps +image/png png +image/prs.btif btif btf +image/prs.pti pti +image/pwg-raster +image/svg+xml svg svgz +image/t38 t38 +image/tiff tiff tif +image/tiff-fx tfx +image/vnd.adobe.photoshop psd +image/vnd.airzip.accelerator.azv azv +image/vnd.cns.inf2 +image/vnd.dece.graphic uvi uvvi uvg uvvg +image/vnd.djvu djvu djv +# sub: text/vnd.dvb.subtitle +image/vnd.dvb.subtitle +image/vnd.dwg dwg +image/vnd.dxf dxf +image/vnd.fastbidsheet fbs +image/vnd.fpx fpx +image/vnd.fst fst +image/vnd.fujixerox.edmics-mmr mmr +image/vnd.fujixerox.edmics-rlc rlc +image/vnd.globalgraphics.pgb pgb +image/vnd.microsoft.icon ico +image/vnd.mix +image/vnd.mozilla.apng apng +image/vnd.ms-modi mdi +image/vnd.net-fpx +image/vnd.radiance hdr rgbe xyze +image/vnd.sealed.png spng spn s1n +image/vnd.sealedmedia.softseal.gif sgif sgi s1g +image/vnd.sealedmedia.softseal.jpg sjpg sjp s1j +image/vnd.svf +image/vnd.tencent.tap tap +image/vnd.valve.source.texture vtf +image/vnd.wap.wbmp wbmp +image/vnd.xiff xif +image/vnd.zbrush.pcx pcx +image/wmf wmf +message/CPIM +message/delivery-status +message/disposition-notification +message/example +message/external-body +message/feedback-report +message/global u8msg +message/global-delivery-status u8dsn +message/global-disposition-notification u8mdn +message/global-headers u8hdr +message/http +# cl: application/simple-filter+xml +message/imdn+xml +# message/news obsoleted by message/rfc822 +message/partial +message/rfc822 eml mail art +message/s-http +message/sip +message/sipfrag +message/tracking-status +message/vnd.si.simp +# wsc: application/vnd.wfa.wsc +message/vnd.wfa.wsc +model/example +model/gltf+json gltf +model/iges igs iges +model/mesh msh mesh silo +model/vnd.collada+xml dae +model/vnd.dwf dwf +# 3dml, 3dm: text/vnd.in3d.3dml +model/vnd.flatland.3dml +model/vnd.gdl gdl gsm win dor lmp rsm msm ism +model/vnd.gs-gdl +model/vnd.gtw gtw +model/vnd.moml+xml moml +model/vnd.mts mts +model/vnd.opengex ogex +model/vnd.parasolid.transmit.binary x_b xmt_bin +model/vnd.parasolid.transmit.text x_t xmt_txt +model/vnd.rosette.annotated-data-model +model/vnd.valve.source.compiled-map bsp +model/vnd.vtu vtu +model/vrml wrl vrml +# x3db: model/x3d+xml +model/x3d+fastinfoset +# x3d: application/vnd.hzn-3d-crossword +model/x3d+xml x3db +model/x3d-vrml x3dv x3dvz +multipart/alternative +multipart/appledouble +multipart/byteranges +multipart/digest +multipart/encrypted +multipart/form-data +multipart/header-set +multipart/mixed +multipart/parallel +multipart/related +multipart/report +multipart/signed +multipart/vnd.bint.med-plus bmed +multipart/voice-message vpm +multipart/x-mixed-replace +text/1d-interleaved-parityfec +text/cache-manifest appcache manifest +text/calendar ics ifb +text/css css +text/csv csv +text/csv-schema csvs +text/directory +text/dns soa zone +text/encaprtp +# text/ecmascript obsoleted by application/ecmascript +text/enriched +text/example +text/fwdred +text/grammar-ref-list +text/html html htm +# text/javascript obsoleted by application/javascript +text/jcr-cnd cnd +text/markdown markdown md +text/mizar miz +text/n3 n3 +text/parameters +text/parityfec +text/plain txt asc text pm el c h cc hh cxx hxx f90 conf log +text/provenance-notation provn +text/prs.fallenstein.rst rst +text/prs.lines.tag tag dsc +text/prs.prop.logic +text/raptorfec +text/RED +text/rfc822-headers +text/richtext rtx +# rtf: application/rtf +text/rtf +text/rtp-enc-aescm128 +text/rtploopback +text/rtx +text/sgml sgml sgm +text/strings +text/t140 +text/tab-separated-values tsv +text/troff t tr roff +text/turtle ttl +text/ulpfec +text/uri-list uris uri +text/vcard vcf vcard +text/vnd.a a +text/vnd.abc abc +text/vnd.ascii-art ascii +# curl: application/vnd.curl +text/vnd.curl +text/vnd.debian.copyright copyright +text/vnd.DMClientScript dms +text/vnd.dvb.subtitle sub +text/vnd.esmertec.theme-descriptor jtd +text/vnd.fly fly +text/vnd.fmi.flexstor flx +text/vnd.graphviz gv dot +text/vnd.in3d.3dml 3dml 3dm +text/vnd.in3d.spot spot spo +text/vnd.IPTC.NewsML +text/vnd.IPTC.NITF +text/vnd.latex-z +text/vnd.motorola.reflex +text/vnd.ms-mediapackage mpf +text/vnd.net2phone.commcenter.command ccc +text/vnd.radisys.msml-basic-layout +text/vnd.si.uricatalogue uric +text/vnd.sun.j2me.app-descriptor jad +text/vnd.trolltech.linguist ts +text/vnd.wap.si si +text/vnd.wap.sl sl +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/xml xml xsd rng +text/xml-external-parsed-entity ent +video/1d-interleaved-parityfec +video/3gpp 3gp 3gpp +video/3gpp2 3g2 3gpp2 +video/3gpp-tt +video/BMPEG +video/BT656 +video/CelB +video/DV +video/encaprtp +video/example +video/H261 +video/H263 +video/H263-1998 +video/H263-2000 +video/H264 +video/H264-RCDO +video/H264-SVC +video/H265 +video/iso.segment m4s +video/JPEG +video/jpeg2000 +video/mj2 mj2 mjp2 +video/MP1S +video/MP2P +video/MP2T +video/mp4 mp4 mpg4 m4v +video/MP4V-ES +video/mpeg mpeg mpg mpe m1v m2v +video/mpeg4-generic +video/MPV +video/nv +video/ogg ogv +video/parityfec +video/pointer +video/quicktime mov qt +video/raptorfec +video/raw +video/rtp-enc-aescm128 +video/rtploopback +video/rtx +video/SMPTE292M +video/ulpfec +video/vc1 +video/vnd.CCTV +video/vnd.dece.hd uvh uvvh +video/vnd.dece.mobile uvm uvvm +video/vnd.dece.mp4 uvu uvvu +video/vnd.dece.pd uvp uvvp +video/vnd.dece.sd uvs uvvs +video/vnd.dece.video uvv uvvv +video/vnd.directv.mpeg +video/vnd.directv.mpeg-tts +video/vnd.dlna.mpeg-tts +video/vnd.dvb.file dvb +video/vnd.fvt fvt +# rm: audio/x-pn-realaudio +video/vnd.hns.video +video/vnd.iptvforum.1dparityfec-1010 +video/vnd.iptvforum.1dparityfec-2005 +video/vnd.iptvforum.2dparityfec-1010 +video/vnd.iptvforum.2dparityfec-2005 +video/vnd.iptvforum.ttsavc +video/vnd.iptvforum.ttsmpeg2 +video/vnd.motorola.video +video/vnd.motorola.videop +video/vnd.mpegurl mxu m4u +video/vnd.ms-playready.media.pyv pyv +video/vnd.nokia.interleaved-multimedia nim +video/vnd.nokia.videovoip +# mp4: video/mp4 +video/vnd.objectvideo +video/vnd.radgamettools.bink bik bk2 +video/vnd.radgamettools.smacker smk +video/vnd.sealed.mpeg1 smpg s11 +# smpg: video/vnd.sealed.mpeg1 +video/vnd.sealed.mpeg4 s14 +video/vnd.sealed.swf sswf ssw +video/vnd.sealedmedia.softseal.mov smov smo s1q +# uvu, uvvu: video/vnd.dece.mp4 +video/vnd.uvvu.mp4 +video/vnd.vivo viv +video/VP8 + +# Non-IANA types + +application/mac-compactpro cpt +application/metalink+xml metalink +application/owl+xml owx +application/rss+xml rss +application/vnd.android.package-archive apk +application/vnd.oma.dd+xml dd +application/vnd.oma.drm.content dcf +# odf: application/vnd.oasis.opendocument.formula +application/vnd.oma.drm.dcf o4a o4v +application/vnd.oma.drm.message dm +application/vnd.oma.drm.rights+wbxml drc +application/vnd.oma.drm.rights+xml dr +application/vnd.sun.xml.calc sxc +application/vnd.sun.xml.calc.template stc +application/vnd.sun.xml.draw sxd +application/vnd.sun.xml.draw.template std +application/vnd.sun.xml.impress sxi +application/vnd.sun.xml.impress.template sti +application/vnd.sun.xml.math sxm +application/vnd.sun.xml.writer sxw +application/vnd.sun.xml.writer.global sxg +application/vnd.sun.xml.writer.template stw +application/vnd.symbian.install sis +application/vnd.wap.mms-message mms +application/x-annodex anx +application/x-bcpio bcpio +application/x-bittorrent torrent +application/x-bzip2 bz2 +application/x-cdlink vcd +application/x-chrome-extension crx +application/x-cpio cpio +application/x-csh csh +application/x-director dcr dir dxr +application/x-dvi dvi +application/x-futuresplash spl +application/x-gtar gtar +application/x-hdf hdf +application/x-java-archive jar +application/x-java-jnlp-file jnlp +application/x-java-pack200 pack +application/x-killustrator kil +application/x-latex latex +application/x-netcdf nc cdf +application/x-perl pl +application/x-rpm rpm +application/x-sh sh +application/x-shar shar +application/x-stuffit sit +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-texinfo texinfo texi +application/x-troff-man man 1 2 3 4 5 6 7 8 +application/x-troff-me me +application/x-troff-ms ms +application/x-ustar ustar +application/x-wais-source src +application/x-xpinstall xpi +application/x-xspf+xml xspf +application/x-xz xz +audio/midi mid midi kar +audio/x-aiff aif aiff aifc +audio/x-annodex axa +audio/x-flac flac +audio/x-matroska mka +audio/x-mod mod ult uni m15 mtm 669 med +audio/x-mpegurl m3u +audio/x-ms-wax wax +audio/x-ms-wma wma +audio/x-pn-realaudio ram rm +audio/x-realaudio ra +audio/x-s3m s3m +audio/x-stm stm +audio/x-wav wav +chemical/x-xyz xyz +image/webp webp +image/x-cmu-raster ras +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-targa tga +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +text/html-sandboxed sandboxed +text/x-pod pod +text/x-setext etx +video/webm webm +video/x-annodex axv +video/x-flv flv +video/x-javafx fxm +video/x-matroska mkv +video/x-matroska-3d mk3d +video/x-ms-asf asx +video/x-ms-wm wm +video/x-ms-wmv wmv +video/x-ms-wmx wmx +video/x-ms-wvx wvx +video/x-msvideo avi +video/x-sgi-movie movie +x-conference/x-cooltalk ice +x-epoc/x-sisx-app sisx diff --git a/cmd/swarm/mru.go b/cmd/swarm/mru.go deleted file mode 100644 index 6176b6d6c8..0000000000 --- a/cmd/swarm/mru.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// Command resource allows the user to create and update signed mutable resource updates -package main - -import ( - "fmt" - "strconv" - "strings" - - "github.com/ethereum/go-ethereum/common/hexutil" - - "github.com/ethereum/go-ethereum/cmd/utils" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/storage/mru" - "gopkg.in/urfave/cli.v1" -) - -func NewGenericSigner(ctx *cli.Context) mru.Signer { - return mru.NewGenericSigner(getPrivKey(ctx)) -} - -// swarm resource create [--name ] [--data <0x Hexdata> [--multihash=false]] -// swarm resource update <0x Hexdata> [--multihash=false] -// swarm resource info - -func resourceCreate(ctx *cli.Context) { - args := ctx.Args() - - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) - multihash = ctx.Bool(SwarmResourceMultihashFlag.Name) - initialData = ctx.String(SwarmResourceDataOnCreateFlag.Name) - name = ctx.String(SwarmResourceNameFlag.Name) - ) - - if len(args) < 1 { - fmt.Println("Incorrect number of arguments") - cli.ShowCommandHelpAndExit(ctx, "create", 1) - return - } - signer := NewGenericSigner(ctx) - frequency, err := strconv.ParseUint(args[0], 10, 64) - if err != nil { - fmt.Printf("Frequency formatting error: %s\n", err.Error()) - cli.ShowCommandHelpAndExit(ctx, "create", 1) - return - } - - metadata := mru.ResourceMetadata{ - Name: name, - Frequency: frequency, - Owner: signer.Address(), - } - - var newResourceRequest *mru.Request - if initialData != "" { - initialDataBytes, err := hexutil.Decode(initialData) - if err != nil { - fmt.Printf("Error parsing data: %s\n", err.Error()) - cli.ShowCommandHelpAndExit(ctx, "create", 1) - return - } - newResourceRequest, err = mru.NewCreateUpdateRequest(&metadata) - if err != nil { - utils.Fatalf("Error creating new resource request: %s", err) - } - newResourceRequest.SetData(initialDataBytes, multihash) - if err = newResourceRequest.Sign(signer); err != nil { - utils.Fatalf("Error signing resource update: %s", err.Error()) - } - } else { - newResourceRequest, err = mru.NewCreateRequest(&metadata) - if err != nil { - utils.Fatalf("Error creating new resource request: %s", err) - } - } - - manifestAddress, err := client.CreateResource(newResourceRequest) - if err != nil { - utils.Fatalf("Error creating resource: %s", err.Error()) - return - } - fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up) - -} - -func resourceUpdate(ctx *cli.Context) { - args := ctx.Args() - - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) - multihash = ctx.Bool(SwarmResourceMultihashFlag.Name) - ) - - if len(args) < 2 { - fmt.Println("Incorrect number of arguments") - cli.ShowCommandHelpAndExit(ctx, "update", 1) - return - } - signer := NewGenericSigner(ctx) - manifestAddressOrDomain := args[0] - data, err := hexutil.Decode(args[1]) - if err != nil { - utils.Fatalf("Error parsing data: %s", err.Error()) - return - } - - // Retrieve resource status and metadata out of the manifest - updateRequest, err := client.GetResourceMetadata(manifestAddressOrDomain) - if err != nil { - utils.Fatalf("Error retrieving resource status: %s", err.Error()) - } - - // set the new data - updateRequest.SetData(data, multihash) - - // sign update - if err = updateRequest.Sign(signer); err != nil { - utils.Fatalf("Error signing resource update: %s", err.Error()) - } - - // post update - err = client.UpdateResource(updateRequest) - if err != nil { - utils.Fatalf("Error updating resource: %s", err.Error()) - return - } -} - -func resourceInfo(ctx *cli.Context) { - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) - ) - args := ctx.Args() - if len(args) < 1 { - fmt.Println("Incorrect number of arguments.") - cli.ShowCommandHelpAndExit(ctx, "info", 1) - return - } - manifestAddressOrDomain := args[0] - metadata, err := client.GetResourceMetadata(manifestAddressOrDomain) - if err != nil { - utils.Fatalf("Error retrieving resource metadata: %s", err.Error()) - return - } - encodedMetadata, err := metadata.MarshalJSON() - if err != nil { - utils.Fatalf("Error encoding metadata to JSON for display:%s", err) - } - fmt.Println(string(encodedMetadata)) -} diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index e2f54c8ffd..680d238d06 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -19,6 +19,7 @@ package main import ( "context" "crypto/ecdsa" + "flag" "fmt" "io/ioutil" "net" @@ -39,8 +40,12 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" + "github.com/ethereum/go-ethereum/swarm/api" + swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" ) +var loglevel = flag.Int("loglevel", 3, "verbosity of logs") + func init() { // Run the app if we've been exec'd as "swarm-test" in runSwarm. reexec.Register("swarm-test", func() { @@ -52,6 +57,20 @@ func init() { }) } +const clusterSize = 3 + +var clusteronce sync.Once +var cluster *testCluster + +func initCluster(t *testing.T) { + clusteronce.Do(func() { + cluster = newTestCluster(t, clusterSize) + }) +} + +func serverFunc(api *api.API) swarmhttp.TestServer { + return swarmhttp.NewServer(api, "") +} func TestMain(m *testing.M) { // check if we have been reexec'd if reexec.Init() { @@ -242,7 +261,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { "--bzzaccount", bzzaccount, "--bzznetworkid", "321", "--bzzport", httpPort, - "--verbosity", "3", + "--verbosity", fmt.Sprint(*loglevel), ) node.Cmd.InputLine(testPassphrase) defer func() { @@ -318,7 +337,7 @@ func newTestNode(t *testing.T, dir string) *testNode { "--bzzaccount", account.Address.String(), "--bzznetworkid", "321", "--bzzport", httpPort, - "--verbosity", "3", + "--verbosity", fmt.Sprint(*loglevel), ) node.Cmd.InputLine(testPassphrase) defer func() { diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go new file mode 100644 index 0000000000..1371d66548 --- /dev/null +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -0,0 +1,320 @@ +package main + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/multihash" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + colorable "github.com/mattn/go-colorable" + "github.com/pborman/uuid" + cli "gopkg.in/urfave/cli.v1" +) + +const ( + feedRandomDataLength = 8 +) + +// TODO: retrieve with manifest + extract repeating code +func cliFeedUploadAndSync(c *cli.Context) error { + + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) + + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) + + generateEndpoints(scheme, cluster, from, to) + + log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing") + + // create a random private key to sign updates with and derive the address + pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") + if err != nil { + return err + } + defer pkFile.Close() + defer os.Remove(pkFile.Name()) + + privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976" + privKey, err := crypto.HexToECDSA(privkeyHex) + if err != nil { + return err + } + user := crypto.PubkeyToAddress(privKey.PublicKey) + userHex := hexutil.Encode(user.Bytes()) + + // save the private key to a file + _, err = io.WriteString(pkFile, privkeyHex) + if err != nil { + return err + } + + // keep hex strings for topic and subtopic + var topicHex string + var subTopicHex string + + // and create combination hex topics for bzz-feed retrieval + // xor'ed with topic (zero-value topic if no topic) + var subTopicOnlyHex string + var mergedSubTopicHex string + + // generate random topic and subtopic and put a hex on them + topicBytes, err := generateRandomData(feed.TopicLength) + topicHex = hexutil.Encode(topicBytes) + subTopicBytes, err := generateRandomData(8) + subTopicHex = hexutil.Encode(subTopicBytes) + if err != nil { + return err + } + mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes) + if err != nil { + return err + } + mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:]) + subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil) + if err != nil { + return err + } + subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:]) + + // create feed manifest, topic only + var out bytes.Buffer + cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic) + } + log.Debug("create topic feed", "manifest", manifestWithTopic) + out.Reset() + + // create feed manifest, subtopic only + cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest subtopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithSubTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic) + } + log.Debug("create subtopic feed", "manifest", manifestWithTopic) + out.Reset() + + // create feed manifest, merged topic + cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + log.Error(err.Error()) + return err + } + manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithMergedTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic) + } + log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic) + out.Reset() + + // create test data + data, err := generateRandomData(feedRandomDataLength) + if err != nil { + return err + } + h := md5.New() + h.Write(data) + dataHash := h.Sum(nil) + dataHex := hexutil.Encode(data) + + // update with topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update topic", "out", out) + out.Reset() + + // update with subtopic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest subtopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update subtopic", "out", out) + out.Reset() + + // update with merged topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest merged topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update mergedtopic", "out", out) + out.Reset() + + time.Sleep(3 * time.Second) + + // retrieve the data + wg := sync.WaitGroup{} + for _, endpoint := range endpoints { + // raw retrieve, topic only + for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { + wg.Add(1) + ruid := uuid.New()[:8] + go func(hex string, endpoint string, ruid string) { + for { + err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) + if err != nil { + continue + } + + wg.Done() + return + } + }(hex, endpoint, ruid) + + } + } + wg.Wait() + log.Info("all endpoints synced random data successfully") + + // upload test file + log.Info("uploading to " + endpoints[0] + " and syncing") + + f, cleanup := generateRandomFile(filesize * 1000) + defer cleanup() + + hash, err := upload(f, endpoints[0]) + if err != nil { + return err + } + hashBytes, err := hexutil.Decode("0x" + hash) + if err != nil { + return err + } + multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes)) + + fileHash, err := digest(f) + if err != nil { + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) + + // update file with topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update topic", "out", out) + out.Reset() + + // update file with subtopic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update subtopic", "out", out) + out.Reset() + + // update file with merged topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update mergedtopic", "out", out) + out.Reset() + + time.Sleep(3 * time.Second) + + for _, endpoint := range endpoints { + + // manifest retrieve, topic only + for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { + wg.Add(1) + ruid := uuid.New()[:8] + go func(url string, endpoint string, ruid string) { + for { + err := fetch(url, endpoint, fileHash, ruid) + if err != nil { + continue + } + + wg.Done() + return + } + }(url, endpoint, ruid) + } + + } + wg.Wait() + log.Info("all endpoints synced random file successfully") + + return nil +} + +func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + log.Trace("sleeping", "ruid", ruid) + time.Sleep(3 * time.Second) + + log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) + res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user) + if err != nil { + return err + } + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 70aee19229..4ff17fd5b7 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -21,7 +21,6 @@ import ( "sort" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" cli "gopkg.in/urfave/cli.v1" ) @@ -34,11 +33,10 @@ var ( filesize int from int to int + verbosity int ) func main() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) app := cli.NewApp() app.Name = "smoke-test" @@ -47,8 +45,8 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ Name: "cluster-endpoint", - Value: "testing", - Usage: "cluster to point to (local, open or testing)", + Value: "prod", + Usage: "cluster to point to (prod or a given namespace)", Destination: &cluster, }, cli.IntFlag{ @@ -80,6 +78,12 @@ func main() { Usage: "file size for generated random file in KB", Destination: &filesize, }, + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity", + Destination: &verbosity, + }, } app.Commands = []cli.Command{ @@ -89,6 +93,12 @@ func main() { Usage: "upload and sync", Action: cliUploadAndSync, }, + { + Name: "feed_sync", + Aliases: []string{"f"}, + Usage: "feed update generate, upload and sync", + Action: cliFeedUploadAndSync, + }, } sort.Sort(cli.FlagsByName(app.Flags)) diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 5e0ff4b0f3..7872421d3d 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -19,7 +19,9 @@ package main import ( "bytes" "crypto/md5" - "crypto/rand" + crand "crypto/rand" + "crypto/tls" + "errors" "fmt" "io" "io/ioutil" @@ -31,6 +33,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" @@ -38,18 +41,13 @@ import ( func generateEndpoints(scheme string, cluster string, from int, to int) { if cluster == "prod" { - cluster = "" - } else if cluster == "local" { for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port)) + endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } - return } else { - cluster = cluster + "." - } - - for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster)) + for port := from; port <= to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster)) + } } if includeLocalhost { @@ -58,6 +56,9 @@ func generateEndpoints(scheme string, cluster string, from int, to int) { } func cliUploadAndSync(c *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, from, to) @@ -85,7 +86,6 @@ func cliUploadAndSync(c *cli.Context) error { wg := sync.WaitGroup{} for _, endpoint := range endpoints { - endpoint := endpoint ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { @@ -112,7 +112,10 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { time.Sleep(3 * time.Second) log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - res, err := http.Get(endpoint + "/bzz:/" + hash + "/") + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + res, err := client.Get(endpoint + "/bzz:/" + hash + "/") if err != nil { log.Warn(err.Error(), "ruid", ruid) return err @@ -166,6 +169,18 @@ func digest(r io.Reader) ([]byte, error) { return h.Sum(nil), nil } +// generates random data in heap buffer +func generateRandomData(datasize int) ([]byte, error) { + b := make([]byte, datasize) + c, err := crand.Read(b) + if err != nil { + return nil, err + } else if c != datasize { + return nil, errors.New("short read") + } + return b, nil +} + // generateRandomFile is creating a temporary file with the requested byte size func generateRandomFile(size int) (f *os.File, teardown func()) { // create a tmp file @@ -181,7 +196,7 @@ func generateRandomFile(size int) (f *os.File, teardown func()) { } buf := make([]byte, size) - _, err = rand.Read(buf) + _, err = crand.Read(buf) if err != nil { panic(err) } diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 9eae2a3f80..992f2d6e97 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -22,34 +22,51 @@ import ( "fmt" "io" "io/ioutil" - "mime" - "net/http" "os" "os/user" "path" "path/filepath" + "strconv" "strings" - "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + + "github.com/ethereum/go-ethereum/cmd/utils" "gopkg.in/urfave/cli.v1" ) -func upload(ctx *cli.Context) { +var upCommand = cli.Command{ + Action: upload, + CustomHelpTemplate: helpTemplate, + Name: "up", + Usage: "uploads a file or directory to swarm using the HTTP API", + ArgsUsage: "", + Flags: []cli.Flag{SwarmEncryptedFlag}, + Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash", +} +func upload(ctx *cli.Context) { args := ctx.Args() var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name) - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) - fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) - mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) - client = swarm.NewClient(bzzapi) - toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) - file string + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name) + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name) + fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) + mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) + client = swarm.NewClient(bzzapi) + toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) + autoDefaultPath = false + file string ) - + if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" { + b, err := strconv.ParseBool(autoDefaultPathString) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err) + } + autoDefaultPath = b + } if len(args) != 1 { if fromStdin { tmp, err := ioutil.TempFile("", "swarm-stdin") @@ -98,6 +115,15 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } + if autoDefaultPath && defaultPath == "" { + defaultEntryCandidate := path.Join(file, "index.html") + log.Debug("trying to find default path", "path", defaultEntryCandidate) + defaultEntryStat, err := os.Stat(defaultEntryCandidate) + if err == nil && !defaultEntryStat.IsDir() { + log.Debug("setting auto detected default path", "path", defaultEntryCandidate) + defaultPath = defaultEntryCandidate + } + } if defaultPath != "" { // construct absolute default path absDefaultPath, _ := filepath.Abs(defaultPath) @@ -118,10 +144,9 @@ func upload(ctx *cli.Context) { return "", fmt.Errorf("error opening file: %s", err) } defer f.Close() - if mimeType == "" { - mimeType = detectMimeType(file) + if mimeType != "" { + f.ContentType = mimeType } - f.ContentType = mimeType return client.Upload(f, "", toEncrypt) } } @@ -138,6 +163,12 @@ func upload(ctx *cli.Context) { // 3. cleans the path, e.g. /a/b/../c -> /a/c // Note, it has limitations, e.g. ~someuser/tmp will not be expanded func expandPath(p string) string { + if i := strings.Index(p, ":"); i > 0 { + return p + } + if i := strings.Index(p, "@"); i > 0 { + return p + } if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { if home := homeDir(); home != "" { p = home + p[1:] @@ -155,19 +186,3 @@ func homeDir() string { } return "" } - -func detectMimeType(file string) string { - if ext := filepath.Ext(file); ext != "" { - return mime.TypeByExtension(ext) - } - f, err := os.Open(file) - if err != nil { - return "" - } - defer f.Close() - buf := make([]byte, 512) - if n, _ := f.Read(buf); n > 0 { - return http.DetectContentType(buf) - } - return "" -} diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index c3199dadc6..616486e37c 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -18,7 +18,6 @@ package main import ( "bytes" - "flag" "fmt" "io" "io/ioutil" @@ -26,72 +25,82 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "testing" "time" "github.com/ethereum/go-ethereum/log" - swarm "github.com/ethereum/go-ethereum/swarm/api/client" - colorable "github.com/mattn/go-colorable" + swarmapi "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/testutil" + "github.com/mattn/go-colorable" ) -var loglevel = flag.Int("loglevel", 3, "verbosity of logs") - func init() { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) } -// TestCLISwarmUp tests that running 'swarm up' makes the resulting file -// available from all nodes via the HTTP API -func TestCLISwarmUp(t *testing.T) { - testCLISwarmUp(false, t) -} -func TestCLISwarmUpRecursive(t *testing.T) { - testCLISwarmUpRecursive(false, t) -} - -// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file -// available from all nodes via the HTTP API -func TestCLISwarmUpEncrypted(t *testing.T) { - testCLISwarmUp(true, t) -} -func TestCLISwarmUpEncryptedRecursive(t *testing.T) { - testCLISwarmUpRecursive(true, t) -} - -func testCLISwarmUp(toEncrypt bool, t *testing.T) { - log.Info("starting 3 node cluster") - cluster := newTestCluster(t, 3) - defer cluster.Shutdown() - - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) +func TestSwarmUp(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() } - defer tmp.Close() - defer os.Remove(tmp.Name()) + + initCluster(t) + + cases := []struct { + name string + f func(t *testing.T) + }{ + {"NoEncryption", testNoEncryption}, + {"Encrypted", testEncrypted}, + {"RecursiveNoEncryption", testRecursiveNoEncryption}, + {"RecursiveEncrypted", testRecursiveEncrypted}, + {"DefaultPathAll", testDefaultPathAll}, + } + + for _, tc := range cases { + t.Run(tc.name, tc.f) + } +} + +// testNoEncryption tests that running 'swarm up' makes the resulting file +// available from all nodes via the HTTP API +func testNoEncryption(t *testing.T) { + testDefault(false, t) +} + +// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file +// available from all nodes via the HTTP API +func testEncrypted(t *testing.T) { + testDefault(true, t) +} + +func testRecursiveNoEncryption(t *testing.T) { + testRecursive(false, t) +} + +func testRecursiveEncrypted(t *testing.T) { + testRecursive(true, t) +} + +func testDefault(toEncrypt bool, t *testing.T) { + tmpFileName := testutil.TempFileWithContent(t, data) + defer os.Remove(tmpFileName) // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - hashRegexp := `[a-f\d]{64}` flags := []string{ "--bzzapi", cluster.Nodes[0].URL, "up", - tmp.Name()} + tmpFileName} if toEncrypt { hashRegexp = `[a-f\d]{128}` flags = []string{ "--bzzapi", cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()} + tmpFileName} } // upload the file with 'swarm up' and expect a hash log.Info(fmt.Sprintf("uploading file with 'swarm up'")) @@ -180,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) { } } -func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { - fmt.Println("starting 3 node cluster") - cluster := newTestCluster(t, 3) - defer cluster.Shutdown() - +func testRecursive(toEncrypt bool, t *testing.T) { tmpUploadDir, err := ioutil.TempDir("", "swarm-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpUploadDir) // create tmp files - data := "notsorandomdata" for _, path := range []string{"tmp1", "tmp2"} { if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil { t.Fatal(err) @@ -231,8 +235,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } defer os.RemoveAll(tmpDownload) bzzLocator := "bzz:/" + hash - flagss := []string{} - flagss = []string{ + flagss := []string{ "--bzzapi", cluster.Nodes[0].URL, "down", "--recursive", @@ -253,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { switch mode := fi.Mode(); { case mode.IsRegular(): - if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil { + if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil { t.Fatalf("encountered an error opening the file returned from the CLI: %v", err) } else { ff := make([]byte, len(data)) @@ -274,19 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } } -// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute +// testDefaultPathAll tests swarm recursive upload with relative and absolute // default paths and with encryption. -func TestCLISwarmUpDefaultPath(t *testing.T) { - testCLISwarmUpDefaultPath(false, false, t) - testCLISwarmUpDefaultPath(false, true, t) - testCLISwarmUpDefaultPath(true, false, t) - testCLISwarmUpDefaultPath(true, true, t) +func testDefaultPathAll(t *testing.T) { + testDefaultPath(false, false, t) + testDefaultPath(false, true, t) + testDefaultPath(true, false, t) + testDefaultPath(true, true, t) } -func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { - cluster := newTestCluster(t, 1) - defer cluster.Shutdown() - +func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") if err != nil { t.Fatal(err) @@ -326,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T up.ExpectExit() hash := matches[0] - client := swarm.NewClient(cluster.Nodes[0].URL) + client := swarmapi.NewClient(cluster.Nodes[0].URL) m, isEncrypted, err := client.DownloadManifest(hash) if err != nil { diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 58d72f32ba..f23aa5775b 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error { // Accumulate the preimages and flush when enough ws gathered preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob) if len(preimages) > 1024 { - rawdb.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, preimages) preimages = make(map[common.Hash][]byte) } } // Flush the last batch preimage data if len(preimages) > 0 { - rawdb.WritePreimages(db, 0, preimages) + rawdb.WritePreimages(db, preimages) } return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 40260bec56..7ec9ce5502 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -300,7 +300,12 @@ var ( CacheDatabaseFlag = cli.IntFlag{ Name: "cache.database", Usage: "Percentage of cache memory allowance to use for database io", - Value: 75, + Value: 50, + } + CacheTrieFlag = cli.IntFlag{ + Name: "cache.trie", + Usage: "Percentage of cache memory allowance to use for trie caching", + Value: 25, } CacheGCFlag = cli.IntFlag{ Name: "cache.gc", @@ -1090,11 +1095,14 @@ func checkExclusive(ctx *cli.Context, args ...interface{}) { if i+1 < len(args) { switch option := args[i+1].(type) { case string: - // Extended flag, expand the name and shift the arguments + // Extended flag check, make sure value set doesn't conflict with passed in option if ctx.GlobalString(flag.GetName()) == option { name += "=" + option + set = append(set, "--"+name) } + // shift arguments and continue i++ + continue case cli.Flag: default: @@ -1159,8 +1167,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { } cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { + cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") @@ -1395,12 +1406,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cache := &core.CacheConfig{ - Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive", - TrieNodeLimit: eth.DefaultConfig.TrieCache, - TrieTimeLimit: eth.DefaultConfig.TrieTimeout, + Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive", + TrieCleanLimit: eth.DefaultConfig.TrieCleanCache, + TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache, + TrieTimeLimit: eth.DefaultConfig.TrieTimeout, + } + if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { + cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)} chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil) diff --git a/common/bytes.go b/common/bytes.go index 0c257a1ee0..c82e616241 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -31,6 +31,15 @@ func ToHex(b []byte) string { return "0x" + hex } +// ToHexArray creates a array of hex-string based on []byte +func ToHexArray(b [][]byte) []string { + r := make([]string, len(b)) + for i := range b { + r[i] = ToHex(b[i]) + } + return r +} + // FromHex returns the bytes represented by the hexadecimal string s. // s may be prefixed with "0x". func FromHex(s string) []byte { diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index f6e8d2e42a..b7c8ec5638 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -31,14 +31,15 @@ import ( var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) -// Contract contains information about a compiled contract, alongside its code. +// Contract contains information about a compiled contract, alongside its code and runtime code. type Contract struct { - Code string `json:"code"` - Info ContractInfo `json:"info"` + Code string `json:"code"` + RuntimeCode string `json:"runtime-code"` + Info ContractInfo `json:"info"` } // ContractInfo contains information about a compiled contract, including access -// to the ABI definition, user and developer docs, and metadata. +// to the ABI definition, source mapping, user and developer docs, and metadata. // // Depending on the source, language version, compiler version, and compiler // options will provide information about how the contract was compiled. @@ -48,6 +49,8 @@ type ContractInfo struct { LanguageVersion string `json:"languageVersion"` CompilerVersion string `json:"compilerVersion"` CompilerOptions string `json:"compilerOptions"` + SrcMap string `json:"srcMap"` + SrcMapRuntime string `json:"srcMapRuntime"` AbiDefinition interface{} `json:"abiDefinition"` UserDoc interface{} `json:"userDoc"` DeveloperDoc interface{} `json:"developerDoc"` @@ -63,14 +66,16 @@ type Solidity struct { // --combined-output format type solcOutput struct { Contracts map[string]struct { - Bin, Abi, Devdoc, Userdoc, Metadata string + BinRuntime string `json:"bin-runtime"` + SrcMapRuntime string `json:"srcmap-runtime"` + Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string } Version string } func (s *Solidity) makeArgs() []string { p := []string{ - "--combined-json", "bin,abi,userdoc,devdoc", + "--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc", "--optimize", // code optimizer switched on } if s.Major > 0 || s.Minor > 4 || s.Patch > 6 { @@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro // provided source, language and compiler version, and compiler options are all // passed through into the Contract structs. // -// The solc output is expected to contain ABI, user docs, and dev docs. +// The solc output is expected to contain ABI, source mapping, user docs, and dev docs. // // Returns an error if the JSON is malformed or missing data, or if the JSON // embedded within the JSON is malformed. @@ -184,13 +189,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin return nil, fmt.Errorf("solc: error reading dev doc: %v", err) } contracts[name] = &Contract{ - Code: "0x" + info.Bin, + Code: "0x" + info.Bin, + RuntimeCode: "0x" + info.BinRuntime, Info: ContractInfo{ Source: source, Language: "Solidity", LanguageVersion: languageVersion, CompilerVersion: compilerVersion, CompilerOptions: compilerOptions, + SrcMap: info.SrcMap, + SrcMapRuntime: info.SrcMapRuntime, AbiDefinition: abi, UserDoc: userdoc, DeveloperDoc: devdoc, diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index eae09f91df..0cb72c35c7 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash { return sigHash(header) } -// Close implements consensus.Engine. It's a noop for clique as there is are no background threads. +// Close implements consensus.Engine. It's a noop for clique as there are no background threads. func (c *Clique) Close() error { return nil } diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go index a04ea235d9..4d8eed4161 100644 --- a/consensus/ethash/api.go +++ b/consensus/ethash/api.go @@ -37,27 +37,28 @@ type API struct { // result[0] - 32 bytes hex encoded current block header pow-hash // result[1] - 32 bytes hex encoded seed hash used for DAG // result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty -func (api *API) GetWork() ([3]string, error) { +// result[3] - hex encoded block number +func (api *API) GetWork() ([4]string, error) { if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { - return [3]string{}, errors.New("not supported") + return [4]string{}, errors.New("not supported") } var ( - workCh = make(chan [3]string, 1) + workCh = make(chan [4]string, 1) errc = make(chan error, 1) ) select { case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}: case <-api.ethash.exitCh: - return [3]string{}, errEthashStopped + return [4]string{}, errEthashStopped } select { case work := <-workCh: return work, nil case err := <-errc: - return [3]string{}, err + return [4]string{}, err } } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d124cb1e20..78892e1da8 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -432,7 +432,7 @@ type hashrate struct { // sealWork wraps a seal work package for remote sealer. type sealWork struct { errc chan error - res chan [3]string + res chan [4]string } // Ethash is a consensus engine based on proof-of-work implementing the ethash diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 8eded2ca81..90cb6470f7 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) { ethash.Seal(nil, block, results, nil) var ( - work [3]string + work [4]string err error ) if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 06c98a7811..3a0919ca99 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -30,6 +30,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { results chan<- *types.Block currentBlock *types.Block - currentWork [3]string + currentWork [4]string notifyTransport = &http.Transport{} notifyClient = &http.Client{ @@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { // result[0], 32 bytes hex encoded current block header pow-hash // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty + // result[3], hex encoded block number makeWork := func(block *types.Block) { hash := ethash.SealHash(block.Header()) currentWork[0] = hash.Hex() currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() + currentWork[3] = hexutil.EncodeBig(block.Number()) // Trace the seal work fetched by remote sealer. currentBlock = block diff --git a/contracts/ens/ens.go b/contracts/ens/ens.go index 75d9d0e4b1..b7448c4717 100644 --- a/contracts/ens/ens.go +++ b/contracts/ens/ens.go @@ -151,6 +151,38 @@ func (self *ENS) Resolve(name string) (common.Hash, error) { return common.BytesToHash(ret[:]), nil } +// Addr is a non-transactional call that returns the address associated with a name. +func (self *ENS) Addr(name string) (common.Address, error) { + node := EnsNode(name) + + resolver, err := self.getResolver(node) + if err != nil { + return common.Address{}, err + } + + ret, err := resolver.Addr(node) + if err != nil { + return common.Address{}, err + } + + return common.BytesToAddress(ret[:]), nil +} + +// SetAddress sets the address associated with a name. Only works if the caller +// owns the name, and the associated resolver implements a `setAddress` function. +func (self *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) { + node := EnsNode(name) + + resolver, err := self.getResolver(node) + if err != nil { + return nil, err + } + + opts := self.TransactOpts + opts.GasLimit = 200000 + return resolver.Contract.SetAddr(&opts, node, addr) +} + // Register registers a new domain name for the caller, making them the owner of the new name. // Only works if the registrar for the parent domain implements the FIFS registrar protocol. func (self *ENS) Register(name string) (*types.Transaction, error) { diff --git a/contracts/ens/ens_test.go b/contracts/ens/ens_test.go index 411b04197c..cd64fbf15f 100644 --- a/contracts/ens/ens_test.go +++ b/contracts/ens/ens_test.go @@ -22,16 +22,18 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" ) var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - name = "my name on ENS" - hash = crypto.Keccak256Hash([]byte("my content")) - addr = crypto.PubkeyToAddress(key.PublicKey) + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + name = "my name on ENS" + hash = crypto.Keccak256Hash([]byte("my content")) + addr = crypto.PubkeyToAddress(key.PublicKey) + testAddr = common.HexToAddress("0x1234123412341234123412341234123412341234") ) func TestENS(t *testing.T) { @@ -74,4 +76,19 @@ func TestENS(t *testing.T) { if vhost != hash { t.Fatalf("resolve error, expected %v, got %v", hash.Hex(), vhost.Hex()) } + + // set the address for the name + if _, err = ens.SetAddr(name, testAddr); err != nil { + t.Fatalf("can't set address: %v", err) + } + contractBackend.Commit() + + // Try to resolve the name to an address + recoveredAddr, err := ens.Addr(name) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if vhost != hash { + t.Fatalf("resolve error, expected %v, got %v", testAddr.Hex(), recoveredAddr.Hex()) + } } diff --git a/core/asm/asm.go b/core/asm/asm.go index ce22f93f92..4257198cc7 100644 --- a/core/asm/asm.go +++ b/core/asm/asm.go @@ -109,9 +109,9 @@ func PrintDisassembled(code string) error { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - fmt.Printf("%06v: %v 0x%x\n", it.PC(), it.Op(), it.Arg()) + fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()) } else { - fmt.Printf("%06v: %v\n", it.PC(), it.Op()) + fmt.Printf("%05x: %v\n", it.PC(), it.Op()) } } return it.Error() @@ -124,9 +124,9 @@ func Disassemble(script []byte) ([]string, error) { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%06v: %v 0x%x\n", it.PC(), it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())) } else { - instrs = append(instrs, fmt.Sprintf("%06v: %v\n", it.PC(), it.Op())) + instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op())) } } if err := it.Error(); err != nil { diff --git a/core/block_validator.go b/core/block_validator.go index 1329f62427..3b9496fece 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { return ErrKnownBlock } - if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { - if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { - return consensus.ErrUnknownAncestor - } - return consensus.ErrPrunedAncestor - } // Header validity is known at this point, check the uncles and transactions header := block.Header() if err := v.engine.VerifyUncles(v.bc, block); err != nil { @@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) } + if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { + if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { + return consensus.ErrUnknownAncestor + } + return consensus.ErrPrunedAncestor + } return nil } diff --git a/core/blockchain.go b/core/blockchain.go index fe961e0c4b..d173b2de29 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -55,6 +55,7 @@ var ( const ( bodyCacheLimit = 256 blockCacheLimit = 256 + receiptsCacheLimit = 32 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 badBlockLimit = 10 @@ -67,9 +68,10 @@ const ( // CacheConfig contains the configuration values for the trie caching/pruning // that's resident in a blockchain. type CacheConfig struct { - Disabled bool // Whether to disable trie write caching (archive node) - TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + Disabled bool // Whether to disable trie write caching (archive node) + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk } // BlockChain represents the canonical chain given a database with a genesis @@ -111,11 +113,12 @@ type BlockChain struct { currentBlock atomic.Value // Current head of the block chain currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) - stateCache state.Database // State database to reuse between imports (contains state cache) - bodyCache *lru.Cache // Cache for the most recent block bodies - bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format - blockCache *lru.Cache // Cache for the most recent entire blocks - futureBlocks *lru.Cache // future blocks are blocks added for later processing + stateCache state.Database // State database to reuse between imports (contains state cache) + bodyCache *lru.Cache // Cache for the most recent block bodies + bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format + receiptsCache *lru.Cache // Cache for the most recent receipts per block + blockCache *lru.Cache // Cache for the most recent entire blocks + futureBlocks *lru.Cache // future blocks are blocks added for later processing quit chan struct{} // blockchain quit channel running int32 // running must be called atomically @@ -138,12 +141,14 @@ type BlockChain struct { func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = &CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, - TrieTimeLimit: 5 * time.Minute, + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, } } bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) + receiptsCache, _ := lru.New(receiptsCacheLimit) blockCache, _ := lru.New(blockCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) badBlocks, _ := lru.New(badBlockLimit) @@ -153,11 +158,12 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par cacheConfig: cacheConfig, db: db, triegc: prque.New(nil), - stateCache: state.NewDatabase(db), + stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit), quit: make(chan struct{}), shouldPreserve: shouldPreserve, bodyCache: bodyCache, bodyRLPCache: bodyRLPCache, + receiptsCache: receiptsCache, blockCache: blockCache, futureBlocks: futureBlocks, engine: engine, @@ -280,6 +286,7 @@ func (bc *BlockChain) SetHead(head uint64) error { // Clear out any stale content from the caches bc.bodyCache.Purge() bc.bodyRLPCache.Purge() + bc.receiptsCache.Purge() bc.blockCache.Purge() bc.futureBlocks.Purge() @@ -388,6 +395,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { return state.New(root, bc.stateCache) } +// StateCache returns the caching database underpinning the blockchain instance. +func (bc *BlockChain) StateCache() state.Database { + return bc.stateCache +} + // Reset purges the entire blockchain, restoring it to its genesis state. func (bc *BlockChain) Reset() error { return bc.ResetWithGenesisBlock(bc.genesisBlock) @@ -549,6 +561,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { return rawdb.HasBody(bc.db, hash, number) } +// HasFastBlock checks if a fast block is fully present in the database or not. +func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool { + if !bc.HasBlock(hash, number) { + return false + } + if bc.receiptsCache.Contains(hash) { + return true + } + return rawdb.HasReceipts(bc.db, hash, number) +} + // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { _, err := bc.stateCache.OpenTrie(hash) @@ -603,11 +626,16 @@ func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { // GetReceiptsByHash retrieves the receipts for all transactions in a given block. func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { + if receipts, ok := bc.receiptsCache.Get(hash); ok { + return receipts.(types.Receipts) + } number := rawdb.ReadHeaderNumber(bc.db, hash) if number == nil { return nil } - return rawdb.ReadReceipts(bc.db, hash, *number) + receipts := rawdb.ReadReceipts(bc.db, hash, *number) + bc.receiptsCache.Add(hash, receipts) + return receipts } // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. @@ -926,7 +954,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() - limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024 + limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 ) if nodes > limit || imgs > 4*1024*1024 { triedb.Cap(limit - ethdb.IdealBatchSize) @@ -990,7 +1018,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } // Write the positional metadata for transaction/receipt lookups and preimages rawdb.WriteTxLookupEntries(batch, block) - rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()) + rawdb.WritePreimages(batch, state.Preimages()) status = CanonStatTy } else { diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 4bdd4ba1c8..1adde1fcb4 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -53,14 +53,14 @@ type ChainIndexerChain interface { // CurrentHeader retrieves the latest locally known header. CurrentHeader() *types.Header - // SubscribeChainEvent subscribes to new head header notifications. - SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription + // SubscribeChainHeadEvent subscribes to new head header notifications. + SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription } // ChainIndexer does a post-processing job for equally sized sections of the // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is // connected to the blockchain through the event system by starting a -// ChainEventLoop in a goroutine. +// ChainHeadEventLoop in a goroutine. // // Further child ChainIndexers can be added which use the output of the parent // section indexer. These child indexers receive new head notifications only @@ -142,8 +142,8 @@ func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) { // cascading background processing. Children do not need to be started, they // are notified about new events by their parents. func (c *ChainIndexer) Start(chain ChainIndexerChain) { - events := make(chan ChainEvent, 10) - sub := chain.SubscribeChainEvent(events) + events := make(chan ChainHeadEvent, 10) + sub := chain.SubscribeChainHeadEvent(events) go c.eventLoop(chain.CurrentHeader(), events, sub) } @@ -190,7 +190,7 @@ func (c *ChainIndexer) Close() error { // eventLoop is a secondary - optional - event loop of the indexer which is only // started for the outermost indexer to push chain head events into a processing // queue. -func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainEvent, sub event.Subscription) { +func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) { // Mark the chain indexer as active, requiring an additional teardown atomic.StoreUint32(&c.active, 1) @@ -219,13 +219,13 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE } header := ev.Block.Header() if header.ParentHash != prevHash { - // Reorg to the common ancestor (might not exist in light sync mode, skip reorg then) + // Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then) // TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly? - // TODO(karalabe): This operation is expensive and might block, causing the event system to - // potentially also lock up. We need to do with on a different thread somehow. - if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { - c.newHead(h.Number.Uint64(), true) + if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash { + if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { + c.newHead(h.Number.Uint64(), true) + } } } c.newHead(header.Number.Uint64(), false) diff --git a/core/chain_makers.go b/core/chain_makers.go index 0bc453fdf3..e3a5537a40 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -33,12 +33,11 @@ import ( // BlockGen creates blocks for testing. // See GenerateChain for a detailed explanation. type BlockGen struct { - i int - parent *types.Block - chain []*types.Block - chainReader consensus.ChainReader - header *types.Header - statedb *state.StateDB + i int + parent *types.Block + chain []*types.Block + header *types.Header + statedb *state.StateDB gasPool *GasPool txs []*types.Transaction @@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) { // For index -1, PrevBlock returns the parent block given to GenerateChain. func (b *BlockGen) PrevBlock(index int) *types.Block { if index >= b.i { - panic("block index out of range") + panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i)) } if index == -1 { return b.parent @@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) { if b.header.Time.Cmp(b.parent.Header().Time) <= 0 { panic("block time out of range") } - b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header()) + chainreader := &fakeChainReader{config: b.config} + b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header()) } // GenerateChain creates a chain of n blocks. The first block's @@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse config = params.TestChainConfig } blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) + chainreader := &fakeChainReader{config: config} genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { - // TODO(karalabe): This is needed for clique, which depends on multiple blocks. - // It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow. - blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil) - defer blockchain.Stop() - - b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine} - b.header = makeHeader(b.chainReader, parent, statedb, b.engine) + b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} + b.header = makeHeader(chainreader, parent, statedb, b.engine) // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { @@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } if b.engine != nil { // Finalize and seal the block - block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts) + block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts) // Write state changes to db root, err := statedb.Commit(config.IsEIP158(b.header.Number)) @@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd }) return blocks } + +type fakeChainReader struct { + config *params.ChainConfig + genesis *types.Block +} + +// Config returns the chain configuration. +func (cr *fakeChainReader) Config() *params.ChainConfig { + return cr.config +} + +func (cr *fakeChainReader) CurrentHeader() *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header { return nil } +func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } +func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block { return nil } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index da54328326..491a125c65 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { } } +// HasReceipts verifies the existence of all the transaction receipts belonging +// to a block. +func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool { + if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { + return false + } + return true +} + // ReadReceipts retrieves all the transaction receipts belonging to a block. func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice @@ -278,7 +287,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece if len(data) == 0 { return nil } - // Convert the revceipts from their storage form to their internal representation + // Convert the receipts from their storage form to their internal representation storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { log.Error("Invalid receipt array RLP", "hash", hash, "err", err) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 514328e87b..3b6e6548d3 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { return data } -// WritePreimages writes the provided set of preimages to the database. `number` is the -// current block number, and is used for debug messages only. -func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { +// WritePreimages writes the provided set of preimages to the database. +func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) { for hash, preimage := range preimages { if err := db.Put(preimageKey(hash), preimage); err != nil { log.Crit("Failed to store trie preimage", "err", err) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index ef597ef309..8a9921ef40 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -35,7 +35,7 @@ var ( // headBlockKey tracks the latest know full block's hash. headBlockKey = []byte("LastBlock") - // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync. + // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") // fastTrieProgressKey tracks the number of trie entries imported during fast sync. diff --git a/core/state/database.go b/core/state/database.go index c1b630991c..f6ea144b9b 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -72,13 +72,19 @@ type Trie interface { } // NewDatabase creates a backing store for state. The returned database is safe for -// concurrent use and retains cached trie nodes in memory. The pool is an optional -// intermediate trie-node memory pool between the low level storage layer and the -// high level trie abstraction. +// concurrent use and retains a few recent expanded trie nodes in memory. To keep +// more historical state in memory, use the NewDatabaseWithCache constructor. func NewDatabase(db ethdb.Database) Database { + return NewDatabaseWithCache(db, 0) +} + +// NewDatabase creates a backing store for state. The returned database is safe for +// concurrent use and retains both a few recent expanded trie nodes in memory, as +// well as a lot of collapsed RLP trie nodes in a large memory cache. +func NewDatabaseWithCache(db ethdb.Database, cache int) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ - db: trie.NewDatabase(db), + db: trie.NewDatabaseWithCache(db, cache), codeSizeCache: csc, } } diff --git a/core/state/statedb.go b/core/state/statedb.go index 216667ce98..76e67d839d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,10 +18,10 @@ package state import ( + "errors" "fmt" "math/big" "sort" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -44,6 +44,13 @@ var ( emptyCode = crypto.Keccak256Hash(nil) ) +type proofList [][]byte + +func (n *proofList) Put(key []byte, value []byte) error { + *n = append(*n, value) + return nil +} + // StateDBs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: @@ -79,8 +86,6 @@ type StateDB struct { journal *journal validRevisions []revision nextRevisionId int - - lock sync.Mutex } // Create a new state from a given trie. @@ -256,6 +261,24 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash return common.Hash{} } +// GetProof returns the MerkleProof for a given Account +func (self *StateDB) GetProof(a common.Address) ([][]byte, error) { + var proof proofList + err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) + return [][]byte(proof), err +} + +// GetProof returns the StorageProof for given key +func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { + var proof proofList + trie := self.StorageTrie(a) + if trie == nil { + return proof, errors.New("storage trie for requested address does not exist") + } + err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + return [][]byte(proof), err +} + // GetCommittedState retrieves a value from the given account's committed storage trie. func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := self.getStateObject(addr) @@ -470,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common // Copy creates a deep, independent copy of the state. // Snapshots of the copied state cannot be applied to the copy. func (self *StateDB) Copy() *StateDB { - self.lock.Lock() - defer self.lock.Unlock() - // Copy all the basic fields, initialize the memory ones state := &StateDB{ db: self.db, diff --git a/core/types/log.go b/core/types/log.go index b629b47ed5..717cd2e5a0 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -47,7 +47,7 @@ type Log struct { TxIndex uint `json:"transactionIndex" gencodec:"required"` // hash of the block in which the transaction was included BlockHash common.Hash `json:"blockHash"` - // index of the log in the receipt + // index of the log in the block Index uint `json:"logIndex" gencodec:"required"` // The Removed field is true if this log was reverted due to a chain reorganisation. diff --git a/core/types/transaction.go b/core/types/transaction.go index 9c6e77be98..7b53cac2c6 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -153,16 +153,21 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error { if err := dec.UnmarshalJSON(input); err != nil { return err } - var V byte - if isProtectedV(dec.V) { - chainID := deriveChainId(dec.V).Uint64() - V = byte(dec.V.Uint64() - 35 - 2*chainID) - } else { - V = byte(dec.V.Uint64() - 27) - } - if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) { - return ErrInvalidSig + + withSignature := dec.V.Sign() != 0 || dec.R.Sign() != 0 || dec.S.Sign() != 0 + if withSignature { + var V byte + if isProtectedV(dec.V) { + chainID := deriveChainId(dec.V).Uint64() + V = byte(dec.V.Uint64() - 35 - 2*chainID) + } else { + V = byte(dec.V.Uint64() - 27) + } + if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) { + return ErrInvalidSig + } } + *tx = Transaction{data: dec} return nil } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index c195d23a32..63132048ee 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -136,7 +136,7 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true) } -// WithSignature returns a new transaction with the given signature. This signature +// SignatureValues returns signature values. This signature // needs to be in the [R || S || V] format where V is 0 or 1. func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) { R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig) @@ -227,13 +227,13 @@ func recoverPlain(sighash common.Hash, R, S, Vb *big.Int, homestead bool) (commo if !crypto.ValidateSignatureValues(V, R, S, homestead) { return common.Address{}, ErrInvalidSig } - // encode the snature in uncompressed format + // encode the signature in uncompressed format r, s := R.Bytes(), S.Bytes() sig := make([]byte, 65) copy(sig[32-len(r):32], r) copy(sig[64-len(s):64], s) sig[64] = V - // recover the public key from the snature + // recover the public key from the signature pub, err := crypto.Ecrecover(sighash[:], sig) if err != nil { return common.Address{}, err diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index b390f45c67..f38d8e7177 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -185,6 +185,7 @@ func TestTransactionJSON(t *testing.T) { } signer := NewEIP155Signer(common.Big1) + transactions := make([]*Transaction, 0, 50) for i := uint64(0); i < 25; i++ { var tx *Transaction switch i % 2 { @@ -193,20 +194,25 @@ func TestTransactionJSON(t *testing.T) { case 1: tx = NewContractCreation(i, common.Big0, 1, common.Big2, []byte("abcdef")) } + transactions = append(transactions, tx) - tx, err := SignTx(tx, signer, key) + signedTx, err := SignTx(tx, signer, key) if err != nil { t.Fatalf("could not sign transaction: %v", err) } + transactions = append(transactions, signedTx) + } + + for _, tx := range transactions { data, err := json.Marshal(tx) if err != nil { - t.Errorf("json.Marshal failed: %v", err) + t.Fatalf("json.Marshal failed: %v", err) } var parsedTx *Transaction if err := json.Unmarshal(data, &parsedTx); err != nil { - t.Errorf("json.Unmarshal failed: %v", err) + t.Fatalf("json.Unmarshal failed: %v", err) } // compare nonce, price, gaslimit, recipient, amount, payload, V, R, S diff --git a/core/vm/analysis.go b/core/vm/analysis.go index f9c4298d39..0ccf47b979 100644 --- a/core/vm/analysis.go +++ b/core/vm/analysis.go @@ -16,34 +16,6 @@ package vm -import ( - "math/big" - - "github.com/ethereum/go-ethereum/common" -) - -// destinations stores one map per contract (keyed by hash of code). -// The maps contain an entry for each location of a JUMPDEST -// instruction. -type destinations map[common.Hash]bitvec - -// has checks whether code has a JUMPDEST at dest. -func (d destinations) has(codehash common.Hash, code []byte, dest *big.Int) bool { - // PC cannot go beyond len(code) and certainly can't be bigger than 63bits. - // Don't bother checking for JUMPDEST in that case. - udest := dest.Uint64() - if dest.BitLen() >= 63 || udest >= uint64(len(code)) { - return false - } - - m, analysed := d[codehash] - if !analysed { - m = codeBitmap(code) - d[codehash] = m - } - return OpCode(code[udest]) == JUMPDEST && m.codeSegment(udest) -} - // bitvec is a bit vector which maps bytes in a program. // An unset bit means the byte is an opcode, a set bit means // it's data (i.e. argument of PUSHxx). diff --git a/core/vm/analysis_test.go b/core/vm/analysis_test.go index a64f90ed9c..fd2d744d87 100644 --- a/core/vm/analysis_test.go +++ b/core/vm/analysis_test.go @@ -16,7 +16,11 @@ package vm -import "testing" +import ( + "testing" + + "github.com/ethereum/go-ethereum/crypto" +) func TestJumpDestAnalysis(t *testing.T) { tests := []struct { @@ -49,5 +53,23 @@ func TestJumpDestAnalysis(t *testing.T) { t.Fatalf("expected %x, got %02x", test.exp, ret[test.which]) } } - +} + +func BenchmarkJumpdestAnalysis_1200k(bench *testing.B) { + // 1.4 ms + code := make([]byte, 1200000) + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + codeBitmap(code) + } + bench.StopTimer() +} +func BenchmarkJumpdestHashing_1200k(bench *testing.B) { + // 4 ms + code := make([]byte, 1200000) + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + crypto.Keccak256Hash(code) + } + bench.StopTimer() } diff --git a/core/vm/contract.go b/core/vm/contract.go index 26bca68951..20baa6e751 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -49,7 +49,8 @@ type Contract struct { caller ContractRef self ContractRef - jumpdests destinations // result of JUMPDEST analysis. + jumpdests map[common.Hash]bitvec // Aggregated result of JUMPDEST analysis. + analysis bitvec // Locally cached result of JUMPDEST analysis Code []byte CodeHash common.Hash @@ -58,21 +59,17 @@ type Contract struct { Gas uint64 value *big.Int - - Args []byte - - DelegateCall bool } // NewContract returns a new contract environment for the execution of EVM. func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uint64) *Contract { - c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object, Args: nil} + c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object} if parent, ok := caller.(*Contract); ok { // Reuse JUMPDEST analysis from parent context if available. c.jumpdests = parent.jumpdests } else { - c.jumpdests = make(destinations) + c.jumpdests = make(map[common.Hash]bitvec) } // Gas should be a pointer so it can safely be reduced through the run @@ -84,10 +81,42 @@ func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uin return c } +func (c *Contract) validJumpdest(dest *big.Int) bool { + udest := dest.Uint64() + // PC cannot go beyond len(code) and certainly can't be bigger than 63bits. + // Don't bother checking for JUMPDEST in that case. + if dest.BitLen() >= 63 || udest >= uint64(len(c.Code)) { + return false + } + // Only JUMPDESTs allowed for destinations + if OpCode(c.Code[udest]) != JUMPDEST { + return false + } + // Do we have a contract hash already? + if c.CodeHash != (common.Hash{}) { + // Does parent context have the analysis? + analysis, exist := c.jumpdests[c.CodeHash] + if !exist { + // Do the analysis and save in parent context + // We do not need to store it in c.analysis + analysis = codeBitmap(c.Code) + c.jumpdests[c.CodeHash] = analysis + } + return analysis.codeSegment(udest) + } + // We don't have the code hash, most likely a piece of initcode not already + // in state trie. In that case, we do an analysis, and save it locally, so + // we don't have to recalculate it for every JUMP instruction in the execution + // However, we don't save it within the parent context + if c.analysis == nil { + c.analysis = codeBitmap(c.Code) + } + return c.analysis.codeSegment(udest) +} + // AsDelegate sets the contract to be a delegate call and returns the current // contract (for chaining calls) func (c *Contract) AsDelegate() *Contract { - c.DelegateCall = true // NOTE: caller must, at all times be a contract. It should never happen // that caller is something other than a Contract. parent := c.caller.(*Contract) @@ -138,12 +167,6 @@ func (c *Contract) Value() *big.Int { return c.value } -// SetCode sets the code to the contract -func (c *Contract) SetCode(hash common.Hash, code []byte) { - c.Code = code - c.CodeHash = hash -} - // SetCallCode sets the code of the contract and address of the backing data // object func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) { @@ -151,3 +174,11 @@ func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []by c.CodeHash = hash c.CodeAddr = addr } + +// SetCodeOptionalHash can be used to provide code, but it's optional to provide hash. +// In case hash is not provided, the jumpdest analysis will not be saved to the parent context +func (c *Contract) SetCodeOptionalHash(addr *common.Address, codeAndHash *codeAndHash) { + c.Code = codeAndHash.code + c.CodeHash = codeAndHash.hash + c.CodeAddr = addr +} diff --git a/core/vm/evm.go b/core/vm/evm.go index fc040c6216..968d2219ea 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -212,12 +212,12 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas evm.StateDB.CreateAccount(addr) } evm.Transfer(evm.StateDB, caller.Address(), to.Address(), value) - // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) + // Even if the account has no code, we need to continue because it might be a precompile start := time.Now() // Capture the tracer start/end events in debug mode @@ -352,8 +352,20 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte return ret, contract.Gas, err } +type codeAndHash struct { + code []byte + hash common.Hash +} + +func (c *codeAndHash) Hash() common.Hash { + if c.hash == (common.Hash{}) { + c.hash = crypto.Keccak256Hash(c.code) + } + return c.hash +} + // create creates a new contract using code as deployment code. -func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.Int, address common.Address) ([]byte, common.Address, uint64, error) { +func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *big.Int, address common.Address) ([]byte, common.Address, uint64, error) { // Depth check execution. Fail if we're trying to execute above the // limit. if evm.depth > int(params.CallCreateDepth) { @@ -382,14 +394,14 @@ func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.I // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, AccountRef(address), value, gas) - contract.SetCallCode(&address, crypto.Keccak256Hash(code), code) + contract.SetCodeOptionalHash(&address, codeAndHash) if evm.vmConfig.NoRecursion && evm.depth > 0 { return nil, address, gas, nil } if evm.vmConfig.Debug && evm.depth == 0 { - evm.vmConfig.Tracer.CaptureStart(caller.Address(), address, true, code, gas, value) + evm.vmConfig.Tracer.CaptureStart(caller.Address(), address, true, codeAndHash.code, gas, value) } start := time.Now() @@ -433,7 +445,7 @@ func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.I // Create creates a new contract using code as deployment code. func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address())) - return evm.create(caller, code, gas, value, contractAddr) + return evm.create(caller, &codeAndHash{code: code}, gas, value, contractAddr) } // Create2 creates a new contract using code as deployment code. @@ -441,8 +453,9 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I // The different between Create2 with Create is Create2 uses sha3(0xff ++ msg.sender ++ salt ++ sha3(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { - contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), code) - return evm.create(caller, code, gas, endowment, contractAddr) + codeAndHash := &codeAndHash{code: code} + contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), codeAndHash.Hash().Bytes()) + return evm.create(caller, codeAndHash, gas, endowment, contractAddr) } // ChainConfig returns the environment's chain configuration diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 10b4f719a7..df79f86eca 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -347,6 +347,17 @@ func gasCreate2(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, if gas, overflow = math.SafeAdd(gas, params.Create2Gas); overflow { return 0, errGasUintOverflow } + wordGas, overflow := bigUint64(stack.Back(2)) + if overflow { + return 0, errGasUintOverflow + } + if wordGas, overflow = math.SafeMul(toWordSize(wordGas), params.Sha3WordGas); overflow { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, wordGas); overflow { + return 0, errGasUintOverflow + } + return gas, nil } diff --git a/core/vm/gen_structlog.go b/core/vm/gen_structlog.go index ade3ca6318..726012e59e 100644 --- a/core/vm/gen_structlog.go +++ b/core/vm/gen_structlog.go @@ -13,20 +13,22 @@ import ( var _ = (*structLogMarshaling)(nil) +// MarshalJSON marshals as JSON. func (s StructLog) MarshalJSON() ([]byte, error) { type StructLog struct { - Pc uint64 `json:"pc"` - Op OpCode `json:"op"` - Gas math.HexOrDecimal64 `json:"gas"` - GasCost math.HexOrDecimal64 `json:"gasCost"` - Memory hexutil.Bytes `json:"memory"` - MemorySize int `json:"memSize"` - Stack []*math.HexOrDecimal256 `json:"stack"` - Storage map[common.Hash]common.Hash `json:"-"` - Depth int `json:"depth"` - Err error `json:"-"` - OpName string `json:"opName"` - ErrorString string `json:"error"` + Pc uint64 `json:"pc"` + Op OpCode `json:"op"` + Gas math.HexOrDecimal64 `json:"gas"` + GasCost math.HexOrDecimal64 `json:"gasCost"` + Memory hexutil.Bytes `json:"memory"` + MemorySize int `json:"memSize"` + Stack []*math.HexOrDecimal256 `json:"stack"` + Storage map[common.Hash]common.Hash `json:"-"` + Depth int `json:"depth"` + RefundCounter uint64 `json:"refund"` + Err error `json:"-"` + OpName string `json:"opName"` + ErrorString string `json:"error"` } var enc StructLog enc.Pc = s.Pc @@ -43,24 +45,27 @@ func (s StructLog) MarshalJSON() ([]byte, error) { } enc.Storage = s.Storage enc.Depth = s.Depth + enc.RefundCounter = s.RefundCounter enc.Err = s.Err enc.OpName = s.OpName() enc.ErrorString = s.ErrorString() return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (s *StructLog) UnmarshalJSON(input []byte) error { type StructLog struct { - Pc *uint64 `json:"pc"` - Op *OpCode `json:"op"` - Gas *math.HexOrDecimal64 `json:"gas"` - GasCost *math.HexOrDecimal64 `json:"gasCost"` - Memory *hexutil.Bytes `json:"memory"` - MemorySize *int `json:"memSize"` - Stack []*math.HexOrDecimal256 `json:"stack"` - Storage map[common.Hash]common.Hash `json:"-"` - Depth *int `json:"depth"` - Err error `json:"-"` + Pc *uint64 `json:"pc"` + Op *OpCode `json:"op"` + Gas *math.HexOrDecimal64 `json:"gas"` + GasCost *math.HexOrDecimal64 `json:"gasCost"` + Memory *hexutil.Bytes `json:"memory"` + MemorySize *int `json:"memSize"` + Stack []*math.HexOrDecimal256 `json:"stack"` + Storage map[common.Hash]common.Hash `json:"-"` + Depth *int `json:"depth"` + RefundCounter *uint64 `json:"refund"` + Err error `json:"-"` } var dec StructLog if err := json.Unmarshal(input, &dec); err != nil { @@ -96,6 +101,9 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { if dec.Depth != nil { s.Depth = *dec.Depth } + if dec.RefundCounter != nil { + s.RefundCounter = *dec.RefundCounter + } if dec.Err != nil { s.Err = dec.Err } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 4d1bd4a342..6696c6e3d1 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/params" ) @@ -124,10 +124,22 @@ func opSmod(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory func opExp(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { base, exponent := stack.pop(), stack.pop() - stack.push(math.Exp(base, exponent)) - - interpreter.intPool.put(base, exponent) - + // some shortcuts + cmpToOne := exponent.Cmp(big1) + if cmpToOne < 0 { // Exponent is zero + // x ^ 0 == 1 + stack.push(base.SetUint64(1)) + } else if base.Sign() == 0 { + // 0 ^ y, if y != 0, == 0 + stack.push(base.SetUint64(0)) + } else if cmpToOne == 0 { // Exponent is one + // x ^ 1 == x + stack.push(base) + } else { + stack.push(math.Exp(base, exponent)) + interpreter.intPool.put(base) + } + interpreter.intPool.put(exponent) return nil, nil } @@ -373,13 +385,20 @@ func opSAR(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory * func opSha3(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() data := memory.Get(offset.Int64(), size.Int64()) - hash := crypto.Keccak256(data) - evm := interpreter.evm - if evm.vmConfig.EnablePreimageRecording { - evm.StateDB.AddPreimage(common.BytesToHash(hash), data) + if interpreter.hasher == nil { + interpreter.hasher = sha3.NewKeccak256().(keccakState) + } else { + interpreter.hasher.Reset() } - stack.push(interpreter.intPool.get().SetBytes(hash)) + interpreter.hasher.Write(data) + interpreter.hasher.Read(interpreter.hasherBuf[:]) + + evm := interpreter.evm + if evm.vmConfig.EnablePreimageRecording { + evm.StateDB.AddPreimage(interpreter.hasherBuf, data) + } + stack.push(interpreter.intPool.get().SetBytes(interpreter.hasherBuf[:])) interpreter.intPool.put(offset, size) return nil, nil @@ -525,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract, // this account should be regarded as a non-existent account and zero should be returned. func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { slot := stack.peek() - slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes()) + address := common.BigToAddress(slot) + if interpreter.evm.StateDB.Empty(address) { + slot.SetUint64(0) + } else { + slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes()) + } return nil, nil } @@ -620,7 +644,7 @@ func opSstore(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memor func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { pos := stack.pop() - if !contract.jumpdests.has(contract.CodeHash, contract.Code, pos) { + if !contract.validJumpdest(pos) { nop := contract.GetOp(pos.Uint64()) return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos) } @@ -633,7 +657,7 @@ func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory func opJumpi(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { pos, cond := stack.pop(), stack.pop() if cond.Sign() != 0 { - if !contract.jumpdests.has(contract.CodeHash, contract.Code, pos) { + if !contract.validJumpdest(pos) { nop := contract.GetOp(pos.Uint64()) return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos) } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 48caadf1f1..8a48d765dd 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -17,10 +17,12 @@ package vm import ( + "bytes" "math/big" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" ) @@ -489,3 +491,99 @@ func BenchmarkOpMstore(bench *testing.B) { } poolOfIntPools.put(evmInterpreter.intPool) } + +func BenchmarkOpSHA3(bench *testing.B) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() + evmInterpreter = NewEVMInterpreter(env, env.vmConfig) + ) + env.interpreter = evmInterpreter + evmInterpreter.intPool = poolOfIntPools.get() + mem.Resize(32) + pc := uint64(0) + start := big.NewInt(0) + + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + stack.pushN(big.NewInt(32), start) + opSha3(&pc, evmInterpreter, nil, mem, stack) + } + poolOfIntPools.put(evmInterpreter.intPool) +} + +func TestCreate2Addreses(t *testing.T) { + type testcase struct { + origin string + salt string + code string + expected string + } + + for i, tt := range []testcase{ + { + origin: "0x0000000000000000000000000000000000000000", + salt: "0x0000000000000000000000000000000000000000", + code: "0x00", + expected: "0x4d1a2e2bb4f88f0250f26ffff098b0b30b26bf38", + }, + { + origin: "0xdeadbeef00000000000000000000000000000000", + salt: "0x0000000000000000000000000000000000000000", + code: "0x00", + expected: "0xB928f69Bb1D91Cd65274e3c79d8986362984fDA3", + }, + { + origin: "0xdeadbeef00000000000000000000000000000000", + salt: "0xfeed000000000000000000000000000000000000", + code: "0x00", + expected: "0xD04116cDd17beBE565EB2422F2497E06cC1C9833", + }, + { + origin: "0x0000000000000000000000000000000000000000", + salt: "0x0000000000000000000000000000000000000000", + code: "0xdeadbeef", + expected: "0x70f2b2914A2a4b783FaEFb75f459A580616Fcb5e", + }, + { + origin: "0x00000000000000000000000000000000deadbeef", + salt: "0xcafebabe", + code: "0xdeadbeef", + expected: "0x60f3f640a8508fC6a86d45DF051962668E1e8AC7", + }, + { + origin: "0x00000000000000000000000000000000deadbeef", + salt: "0xcafebabe", + code: "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + expected: "0x1d8bfDC5D46DC4f61D6b6115972536eBE6A8854C", + }, + { + origin: "0x0000000000000000000000000000000000000000", + salt: "0x0000000000000000000000000000000000000000", + code: "0x", + expected: "0xE33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0", + }, + } { + + origin := common.BytesToAddress(common.FromHex(tt.origin)) + salt := common.BytesToHash(common.FromHex(tt.salt)) + code := common.FromHex(tt.code) + codeHash := crypto.Keccak256(code) + address := crypto.CreateAddress2(origin, salt, codeHash) + /* + stack := newstack() + // salt, but we don't need that for this test + stack.push(big.NewInt(int64(len(code)))) //size + stack.push(big.NewInt(0)) // memstart + stack.push(big.NewInt(0)) // value + gas, _ := gasCreate2(params.GasTable{}, nil, nil, stack, nil, 0) + fmt.Printf("Example %d\n* address `0x%x`\n* salt `0x%x`\n* init_code `0x%x`\n* gas (assuming no mem expansion): `%v`\n* result: `%s`\n\n", i,origin, salt, code, gas, address.String()) + */ + expected := common.BytesToAddress(common.FromHex(tt.expected)) + if !bytes.Equal(expected.Bytes(), address.Bytes()) { + t.Errorf("test %d: expected %s, got %s", i, expected.String(), address.String()) + } + + } +} diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 8e934f60e8..952d96dd4d 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -18,8 +18,10 @@ package vm import ( "fmt" + "hash" "sync/atomic" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/params" ) @@ -68,12 +70,24 @@ type Interpreter interface { CanRun([]byte) bool } +// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports +// Read to get a variable amount of data from the hash state. Read is faster than Sum +// because it doesn't copy the internal state, but also modifies the internal state. +type keccakState interface { + hash.Hash + Read([]byte) (int, error) +} + // EVMInterpreter represents an EVM interpreter type EVMInterpreter struct { evm *EVM cfg Config gasTable params.GasTable - intPool *intPool + + intPool *intPool + + hasher keccakState // Keccak256 hasher instance shared across opcodes + hasherBuf common.Hash // Keccak256 hasher result array shared aross opcodes readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse diff --git a/core/vm/logger.go b/core/vm/logger.go index 85acb8d6d3..1733bf2700 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -56,16 +56,17 @@ type LogConfig struct { // StructLog is emitted to the EVM each cycle and lists information about the current internal state // prior to the execution of the statement. type StructLog struct { - Pc uint64 `json:"pc"` - Op OpCode `json:"op"` - Gas uint64 `json:"gas"` - GasCost uint64 `json:"gasCost"` - Memory []byte `json:"memory"` - MemorySize int `json:"memSize"` - Stack []*big.Int `json:"stack"` - Storage map[common.Hash]common.Hash `json:"-"` - Depth int `json:"depth"` - Err error `json:"-"` + Pc uint64 `json:"pc"` + Op OpCode `json:"op"` + Gas uint64 `json:"gas"` + GasCost uint64 `json:"gasCost"` + Memory []byte `json:"memory"` + MemorySize int `json:"memSize"` + Stack []*big.Int `json:"stack"` + Storage map[common.Hash]common.Hash `json:"-"` + Depth int `json:"depth"` + RefundCounter uint64 `json:"refund"` + Err error `json:"-"` } // overrides for gencodec @@ -177,7 +178,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui storage = l.changedValues[contract.Address()].Copy() } // create a new snaptshot of the EVM. - log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, err} + log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, env.StateDB.GetRefund(), err} l.logs = append(l.logs, log) return nil diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index cdbb70dc42..2ea7535a79 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/params" ) @@ -41,9 +42,15 @@ func (d *dummyContractRef) SetBalance(*big.Int) {} func (d *dummyContractRef) SetNonce(uint64) {} func (d *dummyContractRef) Balance() *big.Int { return new(big.Int) } +type dummyStatedb struct { + state.StateDB +} + +func (*dummyStatedb) GetRefund() uint64 { return 1337 } + func TestStoreCapture(t *testing.T) { var ( - env = NewEVM(Context{}, nil, params.TestChainConfig, Config{}) + env = NewEVM(Context{}, &dummyStatedb{}, params.TestChainConfig, Config{}) logger = NewStructLogger(nil) mem = NewMemory() stack = newstack() @@ -51,9 +58,7 @@ func TestStoreCapture(t *testing.T) { ) stack.push(big.NewInt(1)) stack.push(big.NewInt(0)) - var index common.Hash - logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, contract, 0, nil) if len(logger.changedValues[contract.Address()]) == 0 { t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()])) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index ef664bda30..bac06e524b 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" ) func TestDefaults(t *testing.T) { @@ -148,3 +149,57 @@ func BenchmarkCall(b *testing.B) { } } } +func benchmarkEVM_Create(bench *testing.B, code string) { + var ( + statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) + sender = common.BytesToAddress([]byte("sender")) + receiver = common.BytesToAddress([]byte("receiver")) + ) + + statedb.CreateAccount(sender) + statedb.SetCode(receiver, common.FromHex(code)) + runtimeConfig := Config{ + Origin: sender, + State: statedb, + GasLimit: 10000000, + Difficulty: big.NewInt(0x200000), + Time: new(big.Int).SetUint64(0), + Coinbase: common.Address{}, + BlockNumber: new(big.Int).SetUint64(1), + ChainConfig: ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: new(big.Int), + ByzantiumBlock: new(big.Int), + ConstantinopleBlock: new(big.Int), + DAOForkBlock: new(big.Int), + DAOForkSupport: false, + EIP150Block: new(big.Int), + EIP155Block: new(big.Int), + EIP158Block: new(big.Int), + }, + EVMConfig: vm.Config{}, + } + // Warm up the intpools and stuff + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + Call(receiver, []byte{}, &runtimeConfig) + } + bench.StopTimer() +} + +func BenchmarkEVM_CREATE_500(bench *testing.B) { + // initcode size 500K, repeatedly calls CREATE and then modifies the mem contents + benchmarkEVM_Create(bench, "5b6207a120600080f0600152600056") +} +func BenchmarkEVM_CREATE2_500(bench *testing.B) { + // initcode size 500K, repeatedly calls CREATE2 and then modifies the mem contents + benchmarkEVM_Create(bench, "5b586207a120600080f5600152600056") +} +func BenchmarkEVM_CREATE_1200(bench *testing.B) { + // initcode size 1200K, repeatedly calls CREATE and then modifies the mem contents + benchmarkEVM_Create(bench, "5b62124f80600080f0600152600056") +} +func BenchmarkEVM_CREATE2_1200(bench *testing.B) { + // initcode size 1200K, repeatedly calls CREATE2 and then modifies the mem contents + benchmarkEVM_Create(bench, "5b5862124f80600080f5600152600056") +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 3211957e0a..9b3e76d406 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -77,9 +77,9 @@ func CreateAddress(b common.Address, nonce uint64) common.Address { } // CreateAddress2 creates an ethereum address given the address bytes, initial -// contract code and a salt. -func CreateAddress2(b common.Address, salt [32]byte, code []byte) common.Address { - return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], Keccak256(code))[12:]) +// contract code hash and a salt. +func CreateAddress2(b common.Address, salt [32]byte, inithash []byte) common.Address { + return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], inithash)[12:]) } // ToECDSA creates a private key with the given D value. diff --git a/eth/api.go b/eth/api.go index 708f75a784..816b9cd335 100644 --- a/eth/api.go +++ b/eth/api.go @@ -67,6 +67,15 @@ func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { return hexutil.Uint64(api.e.Miner().HashRate()) } +// ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. +func (api *PublicEthereumAPI) ChainId() hexutil.Uint64 { + chainID := new(big.Int) + if config := api.e.chainConfig; config.IsEIP155(api.e.blockchain.CurrentBlock().Number()) { + chainID = config.ChainID + } + return (hexutil.Uint64)(chainID.Uint64()) +} + // PublicMinerAPI provides an API to control the miner. // It offers only methods that operate on data that pose no security risk when it is publicly accessible. type PublicMinerAPI struct { @@ -435,16 +444,16 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } + triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) + oldTrie, err := trie.NewSecure(startBlock.Root(), triedb, 0) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) + newTrie, err := trie.NewSecure(endBlock.Root(), triedb, 0) if err != nil { return nil, err } - diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{})) iter := trie.NewIterator(diff) diff --git a/eth/api_backend.go b/eth/api_backend.go index 03f6012d78..8748d444fa 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -107,18 +106,11 @@ func (b *EthAPIBackend) GetBlock(ctx context.Context, hash common.Hash) (*types. } func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { - if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { - return rawdb.ReadReceipts(b.eth.chainDb, hash, *number), nil - } - return nil, nil + return b.eth.blockchain.GetReceiptsByHash(hash), nil } func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(b.eth.chainDb, hash, *number) + receipts := b.eth.blockchain.GetReceiptsByHash(hash) if receipts == nil { return nil, nil } diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 5b7f168ec2..2ebbcc5fd9 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -138,7 +138,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl // Ensure we have a valid starting state before doing any work origin := start.NumberU64() - database := state.NewDatabase(api.eth.ChainDb()) + database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis if number := start.NumberU64(); number > 0 { start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1) @@ -391,6 +391,15 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, return api.TraceBlock(ctx, blob, config) } +// TraceBadBlock returns the structured logs created during the execution of a block +// within the blockchain 'badblocks' cache +func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, index int, config *TraceConfig) ([]*txTraceResult, error) { + if blocks := api.eth.blockchain.BadBlocks(); index < len(blocks) { + return api.traceBlock(ctx, blocks[index], config) + } + return nil, fmt.Errorf("index out of range") +} + // traceBlock configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item // per transaction, dependent on the requestd tracer. @@ -483,7 +492,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } // Otherwise try to reexec blocks until we find a state or reach our limit origin := block.NumberU64() - database := state.NewDatabase(api.eth.ChainDb()) + database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) for i := uint64(0); i < reexec; i++ { block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/eth/backend.go b/eth/backend.go index b555b064ad..4721408425 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,7 +154,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EWASMInterpreter: config.EWASMInterpreter, EVMInterpreter: config.EVMInterpreter, } - cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout} + cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieCleanLimit: config.TrieCleanCache, TrieDirtyLimit: config.TrieDirtyCache, TrieTimeLimit: config.TrieTimeout} ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig, eth.shouldPreserve) if err != nil { diff --git a/eth/config.go b/eth/config.go index 1ad95af32e..c1bafc3aa8 100644 --- a/eth/config.go +++ b/eth/config.go @@ -43,15 +43,16 @@ var DefaultConfig = Config{ DatasetsInMem: 1, DatasetsOnDisk: 2, }, - NetworkId: 1, - LightPeers: 100, - DatabaseCache: 768, - TrieCache: 256, - TrieTimeout: 60 * time.Minute, - MinerGasFloor: 8000000, - MinerGasCeil: 8000000, - MinerGasPrice: big.NewInt(params.GWei), - MinerRecommit: 3 * time.Second, + NetworkId: 1, + LightPeers: 100, + DatabaseCache: 512, + TrieCleanCache: 256, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + MinerGasFloor: 8000000, + MinerGasCeil: 8000000, + MinerGasPrice: big.NewInt(params.GWei), + MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ @@ -94,7 +95,8 @@ type Config struct { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCache int + TrieCleanCache int + TrieDirtyCache int TrieTimeout time.Duration // Mining-related options diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index fbde9c6caa..f81a5cbac2 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -60,6 +60,9 @@ var ( maxHeadersProcess = 2048 // Number of header download results to import at once into the chain maxResultsProcess = 2048 // Number of content download results to import at once into the chain + reorgProtThreshold = 48 // Threshold number of recent blocks to disable mini reorg protection + reorgProtHeaderDelay = 2 // Number of headers to delay delivering to cover mini reorgs + fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it @@ -178,6 +181,9 @@ type BlockChain interface { // HasBlock verifies a block's presence in the local chain. HasBlock(common.Hash, uint64) bool + // HasFastBlock verifies a fast block's presence in the local chain. + HasFastBlock(common.Hash, uint64) bool + // GetBlockByHash retrieves a block from the local chain. GetBlockByHash(common.Hash) *types.Block @@ -427,7 +433,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I } height := latest.Number.Uint64() - origin, err := d.findAncestor(p, height) + origin, err := d.findAncestor(p, latest) if err != nil { return err } @@ -584,41 +590,86 @@ func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) { } } +// calculateRequestSpan calculates what headers to request from a peer when trying to determine the +// common ancestor. +// It returns parameters to be used for peer.RequestHeadersByNumber: +// from - starting block number +// count - number of headers to request +// skip - number of headers to skip +// and also returns 'max', the last block which is expected to be returned by the remote peers, +// given the (from,count,skip) +func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { + var ( + from int + count int + MaxCount = MaxHeaderFetch / 16 + ) + // requestHead is the highest block that we will ask for. If requestHead is not offset, + // the highest block that we will get is 16 blocks back from head, which means we + // will fetch 14 or 15 blocks unnecessarily in the case the height difference + // between us and the peer is 1-2 blocks, which is most common + requestHead := int(remoteHeight) - 1 + if requestHead < 0 { + requestHead = 0 + } + // requestBottom is the lowest block we want included in the query + // Ideally, we want to include just below own head + requestBottom := int(localHeight - 1) + if requestBottom < 0 { + requestBottom = 0 + } + totalSpan := requestHead - requestBottom + span := 1 + totalSpan/MaxCount + if span < 2 { + span = 2 + } + if span > 16 { + span = 16 + } + + count = 1 + totalSpan/span + if count > MaxCount { + count = MaxCount + } + if count < 2 { + count = 2 + } + from = requestHead - (count-1)*span + if from < 0 { + from = 0 + } + max := from + (count-1)*span + return int64(from), count, span - 1, uint64(max) +} + // findAncestor tries to locate the common ancestor link of the local chain and // a remote peers blockchain. In the general case when our node was in sync and // on the correct chain, checking the top N links should already get us a match. // In the rare scenario when we ended up on a long reorganisation (i.e. none of // the head links match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) { +func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) { // Figure out the valid ancestor range to prevent rewrite attacks - floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64() + var ( + floor = int64(-1) + localHeight uint64 + remoteHeight = remoteHeader.Number.Uint64() + ) + switch d.mode { + case FullSync: + localHeight = d.blockchain.CurrentBlock().NumberU64() + case FastSync: + localHeight = d.blockchain.CurrentFastBlock().NumberU64() + default: + localHeight = d.lightchain.CurrentHeader().Number.Uint64() + } + p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) + if localHeight >= MaxForkAncestry { + floor = int64(localHeight - MaxForkAncestry) + } + from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) - if d.mode == FullSync { - ceil = d.blockchain.CurrentBlock().NumberU64() - } else if d.mode == FastSync { - ceil = d.blockchain.CurrentFastBlock().NumberU64() - } - if ceil >= MaxForkAncestry { - floor = int64(ceil - MaxForkAncestry) - } - p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) - - // Request the topmost blocks to short circuit binary ancestor lookup - head := ceil - if head > height { - head = height - } - from := int64(head) - int64(MaxHeaderFetch) - if from < 0 { - from = 0 - } - // Span out with 15 block gaps into the future to catch bad head reports - limit := 2 * MaxHeaderFetch / 16 - count := 1 + int((int64(ceil)-from)/16) - if count > limit { - count = limit - } - go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false) + p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip) + go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) // Wait for the remote response to the head fetch number, hash := uint64(0), common.Hash{} @@ -644,9 +695,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return 0, errEmptyHeaderSet } // Make sure the peer's reply conforms to the request - for i := 0; i < len(headers); i++ { - if number := headers[i].Number.Int64(); number != from+int64(i)*16 { - p.log.Warn("Head headers broke chain ordering", "index", i, "requested", from+int64(i)*16, "received", number) + for i, header := range headers { + expectNumber := from + int64(i)*int64((skip+1)) + if number := header.Number.Int64(); number != expectNumber { + p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number) return 0, errInvalidChain } } @@ -654,18 +706,24 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err finished = true for i := len(headers) - 1; i >= 0; i-- { // Skip any headers that underflow/overflow our requested set - if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > ceil { + if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max { continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.blockchain.HasBlock(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) { - number, hash = headers[i].Number.Uint64(), headers[i].Hash() + h := headers[i].Hash() + n := headers[i].Number.Uint64() - // If every header is known, even future ones, the peer straight out lied about its head - if number > height && i == limit-1 { - p.log.Warn("Lied about chain head", "reported", height, "found", number) - return 0, errStallingPeer - } + var known bool + switch d.mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if known { + number, hash = n, h break } } @@ -689,10 +747,12 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err return number, nil } // Ancestor not found, we need to binary search over our chain - start, end := uint64(0), head + start, end := uint64(0), remoteHeight if floor > 0 { start = uint64(floor) } + p.log.Trace("Binary searching for common ancestor", "start", start, "end", end) + for start+1 < end { // Split our chain interval in two, and request the hash to cross check check := (start + end) / 2 @@ -723,16 +783,29 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.blockchain.HasBlock(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) { + h := headers[0].Hash() + n := headers[0].Number.Uint64() + + var known bool + switch d.mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if !known { end = check break } - header := d.lightchain.GetHeaderByHash(headers[0].Hash()) // Independent of sync mode, header surely exists + header := d.lightchain.GetHeaderByHash(h) // Independent of sync mode, header surely exists if header.Number.Uint64() != check { p.log.Debug("Received non requested header", "number", header.Number, "hash", header.Hash(), "request", check) return 0, errBadPeer } start = check + hash = h case <-timeout: p.log.Debug("Waiting for search header timed out", "elapsed", ttl) @@ -843,6 +916,30 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64) } headers = filled[proced:] from += uint64(proced) + } else { + // If we're closing in on the chain head, but haven't yet reached it, delay + // the last few headers so mini reorgs on the head don't cause invalid hash + // chain errors. + if n := len(headers); n > 0 { + // Retrieve the current head we're at + head := uint64(0) + if d.mode == LightSync { + head = d.lightchain.CurrentHeader().Number.Uint64() + } else { + head = d.blockchain.CurrentFastBlock().NumberU64() + if full := d.blockchain.CurrentBlock().NumberU64(); head < full { + head = full + } + } + // If the head is way older than this batch, delay the last few headers + if head+uint64(reorgProtThreshold) < headers[n-1].Number.Uint64() { + delay := reorgProtHeaderDelay + if delay > n { + delay = n + } + headers = headers[:n-delay] + } + } } // Insert all the new headers and fetch the next batch if len(headers) > 0 { @@ -853,8 +950,18 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64) return errCancelHeaderFetch } from += uint64(len(headers)) + getHeaders(from) + } else { + // No headers delivered, or all of them being delayed, sleep a bit and retry + p.log.Trace("All headers delayed, waiting") + select { + case <-time.After(fsHeaderContCheck): + getHeaders(from) + continue + case <-d.cancelCh: + return errCancelHeaderFetch + } } - getHeaders(from) case <-timeout.C: if d.dropPeer == nil { @@ -1205,7 +1312,7 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er } // If no headers were retrieved at all, the peer violated its TD promise that it had a // better chain compared to ours. The only exception is if its promised blocks were - // already imported by other means (e.g. fecher): + // already imported by other means (e.g. fetcher): // // R , L : Both at block 10 // R: Mine block 11, and propagate it to L @@ -1374,7 +1481,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { defer stateSync.Cancel() go func() { if err := stateSync.Wait(); err != nil && err != errCancelStateFetch { - d.queue.Close() // wake up WaitResults + d.queue.Close() // wake up Results } }() // Figure out the ideal pivot block. Note, that this goalpost may move if the @@ -1432,7 +1539,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { defer stateSync.Cancel() go func() { if err := stateSync.Wait(); err != nil && err != errCancelStateFetch { - d.queue.Close() // wake up WaitResults + d.queue.Close() // wake up Results } }() oldPivot = P diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index d1a9a8694b..1a42965d39 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -20,27 +20,20 @@ import ( "errors" "fmt" "math/big" + "strings" "sync" "sync/atomic" "testing" "time" + ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) -) - // Reduce some of the parameters to make the tester faster. func init() { MaxForkAncestry = uint64(10000) @@ -55,6 +48,7 @@ type downloadTester struct { genesis *types.Block // Genesis blocks used by the tester and peers stateDb ethdb.Database // Database used by the tester for syncing from peers peerDb ethdb.Database // Database of the peers containing all data + peers map[string]*downloadTesterPeer ownHashes []common.Hash // Hash chain belonging to the tester ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester @@ -62,129 +56,27 @@ type downloadTester struct { ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain - peerHashes map[string][]common.Hash // Hash chain belonging to different test peers - peerHeaders map[string]map[common.Hash]*types.Header // Headers belonging to different test peers - peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers - peerReceipts map[string]map[common.Hash]types.Receipts // Receipts belonging to different test peers - peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains - - peerMissingStates map[string]map[common.Hash]bool // State entries that fast sync should not return - lock sync.RWMutex } // newTester creates a new downloader test mocker. func newTester() *downloadTester { - testdb := ethdb.NewMemDatabase() - genesis := core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000)) - tester := &downloadTester{ - genesis: genesis, - peerDb: testdb, - ownHashes: []common.Hash{genesis.Hash()}, - ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil}, - ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()}, - peerHashes: make(map[string][]common.Hash), - peerHeaders: make(map[string]map[common.Hash]*types.Header), - peerBlocks: make(map[string]map[common.Hash]*types.Block), - peerReceipts: make(map[string]map[common.Hash]types.Receipts), - peerChainTds: make(map[string]map[common.Hash]*big.Int), - peerMissingStates: make(map[string]map[common.Hash]bool), + genesis: testGenesis, + peerDb: testDB, + peers: make(map[string]*downloadTesterPeer), + ownHashes: []common.Hash{testGenesis.Hash()}, + ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, + ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, + ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, + ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, } tester.stateDb = ethdb.NewMemDatabase() - tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) - + tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) - return tester } -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts, heavy bool) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) { - // Generate the block chain - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), dl.peerDb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If a heavy chain is requested, delay blocks to raise difficulty - if heavy { - block.OffsetTime(-1) - } - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == dl.genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ - ParentHash: block.PrevBlock(i - 1).Hash(), - Number: big.NewInt(block.Number().Int64() - 1), - }) - } - }) - // Convert the block-chain into a hash-chain and header/block maps - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - - headerm := make(map[common.Hash]*types.Header, n+1) - headerm[parent.Hash()] = parent.Header() - - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - - receiptm := make(map[common.Hash]types.Receipts, n+1) - receiptm[parent.Hash()] = parentReceipts - - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - headerm[b.Hash()] = b.Header() - blockm[b.Hash()] = b - receiptm[b.Hash()] = receipts[i] - } - return hashes, headerm, blockm, receiptm -} - -// makeChainFork creates two chains of length n, such that h1[:f] and -// h2[:f] are different but have a common suffix of length n-f. -func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentReceipts types.Receipts, balanced bool) ([]common.Hash, []common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]*types.Block, map[common.Hash]types.Receipts, map[common.Hash]types.Receipts) { - // Create the common suffix - hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false) - - // Create the forks, making the second heavier if non balanced forks were requested - hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false) - hashes1 = append(hashes1, hashes[1:]...) - - heavy := false - if !balanced { - heavy = true - } - hashes2, headers2, blocks2, receipts2 := dl.makeChain(f, 2, blocks[hashes[0]], receipts[hashes[0]], heavy) - hashes2 = append(hashes2, hashes[1:]...) - - for hash, header := range headers { - headers1[hash] = header - headers2[hash] = header - } - for hash, block := range blocks { - blocks1[hash] = block - blocks2[hash] = block - } - for hash, receipt := range receipts { - receipts1[hash] = receipt - receipts2[hash] = receipt - } - return hashes1, hashes2, headers1, headers2, blocks1, blocks2, receipts1, receipts2 -} - // terminate aborts any operations on the embedded downloader and releases all // held resources. func (dl *downloadTester) terminate() { @@ -194,13 +86,10 @@ func (dl *downloadTester) terminate() { // sync starts synchronizing with a remote peer, blocking until it completes. func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { dl.lock.RLock() - hash := dl.peerHashes[id][0] + hash := dl.peers[id].chain.headBlock().Hash() // If no particular TD was requested, load from the peer's blockchain if td == nil { - td = big.NewInt(1) - if diff, ok := dl.peerChainTds[id][hash]; ok { - td = diff - } + td = dl.peers[id].chain.td(hash) } dl.lock.RUnlock() @@ -226,6 +115,15 @@ func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool { return dl.GetBlockByHash(hash) != nil } +// HasFastBlock checks if a block is present in the testers canonical chain. +func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + + _, ok := dl.ownReceipts[hash] + return ok +} + // GetHeader retrieves a header from the testers canonical chain. func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header { dl.lock.RLock() @@ -302,7 +200,7 @@ func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int { } // InsertHeaderChain injects a new batch of headers into the simulated chain. -func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (int, error) { +func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -331,7 +229,7 @@ func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq i } // InsertChain injects a new batch of blocks into the simulated chain. -func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { +func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -346,6 +244,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { dl.ownHeaders[block.Hash()] = block.Header() } dl.ownBlocks[block.Hash()] = block + dl.ownReceipts[block.Hash()] = make(types.Receipts, 0) dl.stateDb.Put(block.Root().Bytes(), []byte{0x00}) dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty()) } @@ -353,7 +252,7 @@ func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { } // InsertReceiptChain injects a new batch of receipts into the simulated chain. -func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (int, error) { +func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (i int, err error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -387,60 +286,13 @@ func (dl *downloadTester) Rollback(hashes []common.Hash) { } // newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error { - return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0) -} - -// newSlowPeer registers a new block download source into the downloader, with a -// specific delay time on processing the network packets sent to it, simulating -// potentially slow network IO. -func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts, delay time.Duration) error { +func (dl *downloadTester) newPeer(id string, version int, chain *testChain) error { dl.lock.Lock() defer dl.lock.Unlock() - var err = dl.downloader.RegisterPeer(id, version, &downloadTesterPeer{dl: dl, id: id, delay: delay}) - if err == nil { - // Assign the owned hashes, headers and blocks to the peer (deep copy) - dl.peerHashes[id] = make([]common.Hash, len(hashes)) - copy(dl.peerHashes[id], hashes) - - dl.peerHeaders[id] = make(map[common.Hash]*types.Header) - dl.peerBlocks[id] = make(map[common.Hash]*types.Block) - dl.peerReceipts[id] = make(map[common.Hash]types.Receipts) - dl.peerChainTds[id] = make(map[common.Hash]*big.Int) - dl.peerMissingStates[id] = make(map[common.Hash]bool) - - genesis := hashes[len(hashes)-1] - if header := headers[genesis]; header != nil { - dl.peerHeaders[id][genesis] = header - dl.peerChainTds[id][genesis] = header.Difficulty - } - if block := blocks[genesis]; block != nil { - dl.peerBlocks[id][genesis] = block - dl.peerChainTds[id][genesis] = block.Difficulty() - } - - for i := len(hashes) - 2; i >= 0; i-- { - hash := hashes[i] - - if header, ok := headers[hash]; ok { - dl.peerHeaders[id][hash] = header - if _, ok := dl.peerHeaders[id][header.ParentHash]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(header.Difficulty, dl.peerChainTds[id][header.ParentHash]) - } - } - if block, ok := blocks[hash]; ok { - dl.peerBlocks[id][hash] = block - if _, ok := dl.peerBlocks[id][block.ParentHash()]; ok { - dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][block.ParentHash()]) - } - } - if receipt, ok := receipts[hash]; ok { - dl.peerReceipts[id][hash] = receipt - } - } - } - return err + peer := &downloadTesterPeer{dl: dl, id: id, chain: chain} + dl.peers[id] = peer + return dl.downloader.RegisterPeer(id, version, peer) } // dropPeer simulates a hard peer removal from the connection pool. @@ -448,89 +300,48 @@ func (dl *downloadTester) dropPeer(id string) { dl.lock.Lock() defer dl.lock.Unlock() - delete(dl.peerHashes, id) - delete(dl.peerHeaders, id) - delete(dl.peerBlocks, id) - delete(dl.peerChainTds, id) - + delete(dl.peers, id) dl.downloader.UnregisterPeer(id) } type downloadTesterPeer struct { - dl *downloadTester - id string - delay time.Duration - lock sync.RWMutex -} - -// setDelay is a thread safe setter for the network delay value. -func (dlp *downloadTesterPeer) setDelay(delay time.Duration) { - dlp.lock.Lock() - defer dlp.lock.Unlock() - - dlp.delay = delay -} - -// waitDelay is a thread safe way to sleep for the configured time. -func (dlp *downloadTesterPeer) waitDelay() { - dlp.lock.RLock() - delay := dlp.delay - dlp.lock.RUnlock() - - time.Sleep(delay) + dl *downloadTester + id string + lock sync.RWMutex + chain *testChain + missingStates map[common.Hash]bool // State entries that fast sync should not return } // Head constructs a function to retrieve a peer's current head hash // and total difficulty. func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - return dlp.dl.peerHashes[dlp.id][0], nil + b := dlp.chain.headBlock() + return b.Hash(), dlp.chain.td(b.Hash()) } // RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - // Find the canonical number of the hash - dlp.dl.lock.RLock() - number := uint64(0) - for num, hash := range dlp.dl.peerHashes[dlp.id] { - if hash == origin { - number = uint64(len(dlp.dl.peerHashes[dlp.id]) - num - 1) - break - } + if reverse { + panic("reverse header requests not supported") } - dlp.dl.lock.RUnlock() - // Use the absolute header fetcher to satisfy the query - return dlp.RequestHeadersByNumber(number, amount, skip, reverse) + result := dlp.chain.headersByHash(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) + return nil } // RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - // Gather the next batch of headers - hashes := dlp.dl.peerHashes[dlp.id] - headers := dlp.dl.peerHeaders[dlp.id] - result := make([]*types.Header, 0, amount) - for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ { - if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok { - result = append(result, header) - } + if reverse { + panic("reverse header requests not supported") } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dlp.dl.downloader.DeliverHeaders(dlp.id, result) - }() + + result := dlp.chain.headersByNumber(origin, amount, skip) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) return nil } @@ -538,24 +349,8 @@ func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, // peer in the download tester. The returned function can be used to retrieve // batches of block bodies from the particularly requested peer. func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - blocks := dlp.dl.peerBlocks[dlp.id] - - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - go dlp.dl.downloader.DeliverBodies(dlp.id, transactions, uncles) - + txs, uncles := dlp.chain.bodies(hashes) + go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles) return nil } @@ -563,21 +358,8 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { - dlp.waitDelay() - - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - receipts := dlp.dl.peerReceipts[dlp.id] - - results := make([][]*types.Receipt, 0, len(hashes)) - for _, hash := range hashes { - if receipt, ok := receipts[hash]; ok { - results = append(results, receipt) - } - } - go dlp.dl.downloader.DeliverReceipts(dlp.id, results) - + receipts := dlp.chain.receipts(hashes) + go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts) return nil } @@ -585,49 +367,46 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { // peer in the download tester. The returned function can be used to retrieve // batches of node state data from the particularly requested peer. func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { - dlp.waitDelay() - dlp.dl.lock.RLock() defer dlp.dl.lock.RUnlock() results := make([][]byte, 0, len(hashes)) for _, hash := range hashes { if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { - if !dlp.dl.peerMissingStates[dlp.id][hash] { + if !dlp.missingStates[hash] { results = append(results, data) } } } go dlp.dl.downloader.DeliverNodeData(dlp.id, results) - return nil } // assertOwnChain checks if the local chain contains the correct number of items // of the various chain components. func assertOwnChain(t *testing.T, tester *downloadTester, length int) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + assertOwnForkedChain(t, tester, 1, []int{length}) } // assertOwnForkedChain checks if the local forked chain contains the correct // number of items of the various chain components. func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) { - // Initialize the counters for the first fork - headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + // Initialize the counters for the first fork + headers, blocks, receipts := lengths[0], lengths[0], lengths[0] - if receipts < 0 { - receipts = 1 - } // Update the counters for each subsequent fork for _, length := range lengths[1:] { headers += length - common blocks += length - common - receipts += length - common - fsMinFullBlocks + receipts += length - common } - switch tester.downloader.mode { - case FullSync: - receipts = 1 - case LightSync: + if tester.downloader.mode == LightSync { blocks, receipts = 1, 1 } if hs := len(tester.ownHeaders); hs != headers { @@ -639,21 +418,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng if rs := len(tester.ownReceipts); rs != receipts { t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts) } - // Verify the state trie too for fast syncs - /*if tester.downloader.mode == FastSync { - pivot := uint64(0) - var index int - if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common { - index = pivot - } else { - index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot) - } - if index > 0 { - if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil { - t.Fatalf("state reconstruction failed: %v", err) - } - } - }*/ } // Tests that simple synchronization against a canonical chain works correctly. @@ -673,16 +437,14 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(blockCacheItems - 15) + tester.newPeer("peer", protocol, chain) // Synchronise with the peer and make sure all relevant data was retrieved if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if a large batch of blocks are being downloaded, it is throttled @@ -699,10 +461,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long block chain to download and the tester - targetBlocks := 8 * blockCacheItems - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + targetBlocks := testChainBase.len() - 1 + tester.newPeer("peer", protocol, testChainBase) // Wrap the importer to allow stepping blocked, proceed := uint32(0), make(chan struct{}) @@ -734,9 +494,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { cached = len(tester.downloader.queue.blockDonePool) if mode == FastSync { if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached { - //if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot { cached = receipts - //} } } frozen = int(atomic.LoadUint32(&blocked)) @@ -744,7 +502,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { tester.downloader.queue.lock.Unlock() tester.lock.Unlock() - if cached == blockCacheItems || retrieved+cached+frozen == targetBlocks+1 { + if cached == blockCacheItems || cached == blockCacheItems-reorgProtHeaderDelay || retrieved+cached+frozen == targetBlocks+1 || retrieved+cached+frozen == targetBlocks+1-reorgProtHeaderDelay { break } } @@ -754,7 +512,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { tester.lock.RLock() retrieved = len(tester.ownBlocks) tester.lock.RUnlock() - if cached != blockCacheItems && retrieved+cached+frozen != targetBlocks+1 { + if cached != blockCacheItems && cached != blockCacheItems-reorgProtHeaderDelay && retrieved+cached+frozen != targetBlocks+1 && retrieved+cached+frozen != targetBlocks+1-reorgProtHeaderDelay { t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheItems, retrieved, frozen, targetBlocks+1) } // Permit the blocked blocks to import @@ -786,24 +544,22 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkLightB.shorten(testChainBase.len() + 80) + tester.newPeer("fork A", protocol, chainA) + tester.newPeer("fork B", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("fork A", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("fork B", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that synchronising against a much shorter but much heavyer fork works @@ -821,24 +577,22 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := MaxHashFetch, 4*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("light", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy", protocol, hashesB[fork/2:], headersB, blocksB, receiptsB) + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkHeavy.shorten(testChainBase.len() + 80) + tester.newPeer("light", protocol, chainA) + tester.newPeer("heavy", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("light", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and make sure that fork is pulled too if err := tester.sync("heavy", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } // Tests that chain forks are contained within a certain interval of the current @@ -857,18 +611,16 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("rewriter", protocol, hashesB, headersB, blocksB, receiptsB) + chainA := testChainForkLightA + chainB := testChainForkLightB + tester.newPeer("original", protocol, chainA) + tester.newPeer("rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { @@ -893,17 +645,16 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a long enough forked chain - common, fork := 13, int(MaxForkAncestry+17) - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, false) - - tester.newPeer("original", protocol, hashesA, headersA, blocksA, receiptsA) - tester.newPeer("heavy-rewriter", protocol, hashesB[MaxForkAncestry-17:], headersB, blocksB, receiptsB) // Root the fork below the ancestor limit + chainA := testChainForkLightA + chainB := testChainForkHeavy + tester.newPeer("original", protocol, chainA) + tester.newPeer("heavy-rewriter", protocol, chainB) // Synchronise with the peer and make sure all blocks were retrieved if err := tester.sync("original", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, common+fork+1) + assertOwnChain(t, tester, chainA.len()) // Synchronise with the second peer and ensure that the fork is rejected to being too old if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { @@ -924,7 +675,7 @@ func TestInactiveDownloader62(t *testing.T) { t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) } } @@ -962,17 +713,8 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download and the tester - targetBlocks := blockCacheItems - 15 - if targetBlocks >= MaxHashFetch { - targetBlocks = MaxHashFetch - 15 - } - if targetBlocks >= MaxHeaderFetch { - targetBlocks = MaxHeaderFetch - 15 - } - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase.shorten(MaxHeaderFetch) + tester.newPeer("peer", protocol, chain) // Make sure canceling works with a pristine downloader tester.downloader.Cancel() @@ -1005,17 +747,16 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Create various peers with various parts of the chain targetPeers := 8 - targetBlocks := targetPeers*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetPeers * 100) for i := 0; i < targetPeers; i++ { id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts) + tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1))) } if err := tester.sync("peer #0", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that synchronisations behave well in multi-version protocol environments @@ -1034,24 +775,23 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Create peers of every type - tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) - tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) - tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) + tester.newPeer("peer 62", 62, chain) + tester.newPeer("peer 63", 63, chain) + tester.newPeer("peer 64", 64, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peerHashes[peer]; !ok { + if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) } } @@ -1073,10 +813,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { defer tester.terminate() // Create a block chain to download - targetBlocks := 2*blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) + chain := testChainBase + tester.newPeer("peer", protocol, chain) // Instrument the downloader to signal body requests bodiesHave, receiptsHave := int32(0), int32(0) @@ -1090,16 +828,16 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("peer", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) // Validate the number of block bodies that should have been requested bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range blocks { + for _, block := range chain.blockm { if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { bodiesNeeded++ } } - for _, receipt := range receipts { + for _, receipt := range chain.receiptm { if mode == FastSync && len(receipt) > 0 { receiptsNeeded++ } @@ -1127,24 +865,20 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) - - // Attempt a full sync with an attacker feeding gapped headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["attack"], hashes[missing]) + chain := testChainBase.shorten(blockCacheItems - 15) + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that if requested headers are shifted (i.e. first is missing), the queue @@ -1162,25 +896,24 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Attempt a full sync with an attacker feeding shifted headers - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - delete(tester.peerHeaders["attack"], hashes[len(hashes)-2]) - delete(tester.peerBlocks["attack"], hashes[len(hashes)-2]) - delete(tester.peerReceipts["attack"], hashes[len(hashes)-2]) - + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[1]) + delete(brokenChain.blockm, brokenChain.chain[1]) + delete(brokenChain.receiptm, brokenChain.chain[1]) + tester.newPeer("attack", protocol, brokenChain) if err := tester.sync("attack", nil, mode); err == nil { t.Fatalf("succeeded attacker synchronisation") } + // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - assertOwnChain(t, tester, targetBlocks+1) + assertOwnChain(t, tester, chain.len()) } // Tests that upon detecting an invalid header, the recent ones are rolled back @@ -1198,13 +931,14 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Create a small enough block chain to download targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(targetBlocks) // Attempt to sync with an attacker that feeds junk during the fast sync phase. // This should result in the last fsHeaderSafetyNet headers being rolled back. - tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts) missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) + fastAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) + tester.newPeer("fast-attack", protocol, fastAttackChain) if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") @@ -1212,13 +946,15 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } + // Attempt to sync with an attacker that feeds junk during the block import phase. // This should result in both the last fsHeaderSafetyNet number of headers being // rolled back, and also the pivot point being reverted to a non-block status. - tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts) missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in - delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing]) + blockAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in + delete(blockAttackChain.headerm, blockAttackChain.chain[missing]) + tester.newPeer("block-attack", protocol, blockAttackChain) if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") @@ -1231,19 +967,18 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } + // Attempt to sync with an attacker that withholds promised blocks after the // fast sync pivot point. This could be a trial to leave the node with a bad // but already imported pivot block. - tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts) - missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - + withholdAttackChain := chain.shorten(chain.len()) + tester.newPeer("withhold-attack", protocol, withholdAttackChain) tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i <= len(hashes); i++ { - delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i]) + for i := missing; i < withholdAttackChain.len(); i++ { + delete(withholdAttackChain.headerm, withholdAttackChain.chain[i]) } tester.downloader.syncInitHook = nil } - if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } @@ -1255,20 +990,21 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { t.Errorf("fast sync pivot block #%d not rolled back", head) } } - // Synchronise with the valid peer and make sure sync succeeds. Since the last - // rollback should also disable fast syncing for this process, verify that we - // did a fresh full sync. Note, we can't assert anything about the receipts - // since we won't purge the database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + + // synchronise with the valid peer and make sure sync succeeds. Since the last rollback + // should also disable fast syncing for this process, verify that we did a fresh full + // sync. Note, we can't assert anything about the receipts since we won't purge the + // database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, chain) if err := tester.sync("valid", nil, mode); err != nil { t.Fatalf("failed to synchronise blocks: %v", err) } - if hs := len(tester.ownHeaders); hs != len(headers) { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers)) + if hs := len(tester.ownHeaders); hs != chain.len() { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len()) } if mode != LightSync { - if bs := len(tester.ownBlocks); bs != len(blocks) { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks)) + if bs := len(tester.ownBlocks); bs != chain.len() { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len()) } } } @@ -1288,9 +1024,8 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - hashes, headers, blocks, receipts := tester.makeChain(0, 0, tester.genesis, nil, false) - tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts) - + chain := testChainBase.shorten(1) + tester.newPeer("attack", protocol, chain) if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) } @@ -1333,21 +1068,22 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Run the tests and check disconnection status tester := newTester() defer tester.terminate() + chain := testChainBase.shorten(1) for i, tt := range tests { // Register a new peer and ensure it's presence id := fmt.Sprintf("test %d", i) - if err := tester.newPeer(id, protocol, []common.Hash{tester.genesis.Hash()}, nil, nil, nil); err != nil { + if err := tester.newPeer(id, protocol, chain); err != nil { t.Fatalf("test %d: failed to register new peer: %v", i, err) } - if _, ok := tester.peerHashes[id]; !ok { + if _, ok := tester.peers[id]; !ok { t.Fatalf("test %d: registered peer not found", i) } // Simulate a synchronisation and check the required result tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync) - if _, ok := tester.peerHashes[id]; !ok != tt.drop { + if _, ok := tester.peers[id]; !ok != tt.drop { t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) } } @@ -1367,10 +1103,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1380,12 +1113,10 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) + tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2)) pending := new(sync.WaitGroup) pending.Add(1) @@ -1396,16 +1127,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chain.len()/2 - 1), + }) progress <- struct{}{} pending.Wait() // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts) + tester.newPeer("peer-full", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("peer-full", nil, mode); err != nil { @@ -1413,15 +1143,31 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len()/2 - 1), + HighestBlock: uint64(chain.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) +} + +func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + p := d.Progress() + p.KnownStates, p.PulledStates = 0, 0 + want.KnownStates, want.PulledStates = 0, 0 + if p != want { + t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) } } @@ -1440,10 +1186,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a forked chain to simulate origin revertal - common, fork := MaxHashFetch, 2*MaxHashFetch - hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := tester.makeChainFork(common+fork, fork, tester.genesis, nil, true) + chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHashFetch) + chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHashFetch) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1453,15 +1197,12 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) + tester.newPeer("fork A", protocol, chainA) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork A", nil, mode); err != nil { @@ -1469,9 +1210,10 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) - } + + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chainA.len() - 1), + }) progress <- struct{}{} pending.Wait() @@ -1479,9 +1221,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB) + tester.newPeer("fork B", protocol, chainB) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("fork B", nil, mode); err != nil { @@ -1489,16 +1230,20 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainA.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainB.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) } // Tests that if synchronisation is aborted due to some failure, then the progress @@ -1516,10 +1261,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small enough block chain to download - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) @@ -1529,20 +1271,18 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + // Attempt a full sync with a faulty peer - tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) - missing := targetBlocks / 2 - delete(tester.peerHeaders["faulty"], hashes[missing]) - delete(tester.peerBlocks["faulty"], hashes[missing]) - delete(tester.peerReceipts["faulty"], hashes[missing]) + brokenChain := chain.shorten(chain.len()) + missing := brokenChain.len() / 2 + delete(brokenChain.headerm, brokenChain.chain[missing]) + delete(brokenChain.blockm, brokenChain.chain[missing]) + delete(brokenChain.receiptm, brokenChain.chain[missing]) + tester.newPeer("faulty", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("faulty", nil, mode); err == nil { @@ -1550,16 +1290,17 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress origin remind the same after a failure - tester.newPeer("valid", protocol, hashes, headers, blocks, receipts) + // Synchronise with a good peer and check that the progress origin remind the same + // after a failure + tester.newPeer("valid", protocol, chain) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("valid", nil, mode); err != nil { @@ -1567,16 +1308,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) - } - progress <- struct{}{} - pending.Wait() + checkProgress(t, tester.downloader, "completing", afterFailedSync) // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) - } + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) } // Tests that if an attacker fakes a chain height, after the attack is detected, @@ -1593,34 +1333,27 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - - // Create a small block chain - targetBlocks := blockCacheItems - 15 - hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false) + chain := testChainBase.shorten(blockCacheItems - 15) // Set a sync init hook to catch progress changes starting := make(chan struct{}) progress := make(chan struct{}) - tester.downloader.syncInitHook = func(origin, latest uint64) { starting <- struct{}{} <-progress } - // Retrieve the sync progress and ensure they are zero (pristine sync) - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) - } - // Create and sync with an attacker that promises a higher chain than available - tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) - for i := 1; i < 3; i++ { - delete(tester.peerHeaders["attack"], hashes[i]) - delete(tester.peerBlocks["attack"], hashes[i]) - delete(tester.peerReceipts["attack"], hashes[i]) + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Create and sync with an attacker that promises a higher chain than available. + brokenChain := chain.shorten(chain.len()) + numMissing := 5 + for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- { + delete(brokenChain.headerm, brokenChain.chain[i]) } + tester.newPeer("attack", protocol, brokenChain) pending := new(sync.WaitGroup) pending.Add(1) - go func() { defer pending.Done() if err := tester.sync("attack", nil, mode); err == nil { @@ -1628,14 +1361,17 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) - } + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) progress <- struct{}{} pending.Wait() + afterFailedSync := tester.downloader.Progress() - // Synchronise with a good peer and check that the progress height has been reduced to the true value - tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts) + // Synchronise with a good peer and check that the progress height has been reduced to + // the true value. + validChain := chain.shorten(chain.len() - numMissing) + tester.newPeer("valid", protocol, validChain) pending.Add(1) go func() { @@ -1645,23 +1381,25 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + CurrentBlock: afterFailedSync.CurrentBlock, + HighestBlock: uint64(validChain.len() - 1), + }) + + // Check final progress after successful sync. progress <- struct{}{} pending.Wait() - - // Check final progress after successful sync - if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) - } + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(validChain.len() - 1), + HighestBlock: uint64(validChain.len() - 1), + }) } // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -// We use data driven subtests to manage this so that it will be parallel on its own -// and not with the other tests, avoiding intermittent failures. func TestDeliverHeadersHang(t *testing.T) { + t.Parallel() + testCases := []struct { protocol int syncMode SyncMode @@ -1675,15 +1413,38 @@ func TestDeliverHeadersHang(t *testing.T) { } for _, tc := range testCases { t.Run(fmt.Sprintf("protocol %d mode %v", tc.protocol, tc.syncMode), func(t *testing.T) { + t.Parallel() testDeliverHeadersHang(t, tc.protocol, tc.syncMode) }) } } +func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { + master := newTester() + defer master.terminate() + chain := testChainBase.shorten(15) + + for i := 0; i < 200; i++ { + tester := newTester() + tester.peerDb = master.peerDb + tester.newPeer("peer", protocol, chain) + + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + } +} + type floodingTestPeer struct { peer Peer tester *downloadTester - pend sync.WaitGroup } func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } @@ -1702,23 +1463,29 @@ func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { deliveriesDone := make(chan struct{}, 500) - for i := 0; i < cap(deliveriesDone); i++ { + for i := 0; i < cap(deliveriesDone)-1; i++ { peer := fmt.Sprintf("fake-peer%d", i) - ftp.pend.Add(1) - go func() { ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) deliveriesDone <- struct{}{} - ftp.pend.Done() }() } - // Deliver the actual requested headers. - go ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + // None of the extra deliveries should block. timeout := time.After(60 * time.Second) + launched := false for i := 0; i < cap(deliveriesDone); i++ { select { case <-deliveriesDone: + if !launched { + // Start delivering the requested headers + // after one of the flooding responses has arrived. + go func() { + ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + deliveriesDone <- struct{}{} + }() + launched = true + } case <-timeout: panic("blocked") } @@ -1726,30 +1493,77 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int return nil } -func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { - t.Parallel() - - master := newTester() - defer master.terminate() - - hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) - for i := 0; i < 200; i++ { - tester := newTester() - tester.peerDb = master.peerDb - - tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, +func TestRemoteHeaderRequestSpan(t *testing.T) { + testCases := []struct { + remoteHeight uint64 + localHeight uint64 + expected []int + }{ + // Remote is way higher. We should ask for the remote head and go backwards + {1500, 1000, + []int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499}, + }, + {15000, 13006, + []int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999}, + }, + //Remote is pretty close to us. We don't have to fetch as many + {1200, 1150, + []int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199}, + }, + // Remote is equal to us (so on a fork with higher td) + // We should get the closest couple of ancestors + {1500, 1500, + []int{1497, 1499}, + }, + // We're higher than the remote! Odd + {1000, 1500, + []int{997, 999}, + }, + // Check some weird edgecases that it behaves somewhat rationally + {0, 1500, + []int{0, 2}, + }, + {6000000, 0, + []int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999}, + }, + {0, 0, + []int{0, 2}, + }, + } + reqs := func(from, count, span int) []int { + var r []int + num := from + for len(r) < count { + r = append(r, num) + num += span + 1 } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() + return r + } + for i, tt := range testCases { + from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight) + data := reqs(int(from), count, span) - // Flush all goroutines to prevent messing with subsequent tests - tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait() + if max != uint64(data[len(data)-1]) { + t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max) + } + failed := false + if len(data) != len(tt.expected) { + failed = true + t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data)) + } else { + for j, n := range data { + if n != tt.expected[j] { + failed = true + break + } + } + } + if failed { + res := strings.Replace(fmt.Sprint(data), " ", ",", -1) + exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1) + fmt.Printf("got: %v\n", res) + fmt.Printf("exp: %v\n", exp) + t.Errorf("test %d: wrong values", i) + } } } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 428a60f8a1..60f86d0e14 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -229,13 +229,6 @@ func (p *peerConnection) SetHeadersIdle(delivered int) { p.setIdle(p.headerStarted, delivered, &p.headerThroughput, &p.headerIdle) } -// SetBlocksIdle sets the peer to idle, allowing it to execute new block retrieval -// requests. Its estimated block retrieval throughput is updated with that measured -// just now. -func (p *peerConnection) SetBlocksIdle(delivered int) { - p.setIdle(p.blockStarted, delivered, &p.blockThroughput, &p.blockIdle) -} - // SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval // requests. Its estimated body retrieval throughput is updated with that measured // just now. diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index a0e8a6d48c..7c33953811 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -143,7 +143,7 @@ func (q *queue) Reset() { q.resultOffset = 0 } -// Close marks the end of the sync, unblocking WaitResults. +// Close marks the end of the sync, unblocking Results. // It may be called even if the queue is already closed. func (q *queue) Close() { q.lock.Lock() @@ -325,7 +325,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { } // Make sure no duplicate requests are executed if _, ok := q.blockTaskPool[hash]; ok { - log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) + log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) continue } if _, ok := q.receiptTaskPool[hash]; ok { @@ -545,7 +545,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common taskQueue.Push(header, -int64(header.Number.Uint64())) } if progress { - // Wake WaitResults, resultCache was modified + // Wake Results, resultCache was modified q.active.Signal() } // Assemble and return the block download request @@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, } // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) + + // Remove the expired requests from the pending pool directly + delete(pendPool, id) } } - // Remove the expired requests from the pending pool - for id := range expiries { - delete(pendPool, id) - } return expiries } @@ -857,7 +856,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ taskQueue.Push(header, -int64(header.Number.Uint64())) } } - // Wake up WaitResults + // Wake up Results if accepted > 0 { q.active.Signal() } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 8d33dfec74..29d5ee4ddb 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) { s.d.dropPeer(req.peer.id) } // Process all the received blobs and check for stale delivery - if err = s.process(req); err != nil { + delivered, err := s.process(req) + if err != nil { log.Warn("Node data write error", "err", err) return err } - req.peer.SetNodeDataIdle(len(req.response)) + req.peer.SetNodeDataIdle(delivered) } } return nil @@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) { // process iterates over a batch of delivered state data, injecting each item // into a running state sync, re-queuing any items that were requested but not // delivered. -func (s *stateSync) process(req *stateReq) error { +// Returns whether the peer actually managed to deliver anything of value, +// and any error that occurred +func (s *stateSync) process(req *stateReq) (int, error) { // Collect processing stats and update progress if valid data was received - duplicate, unexpected := 0, 0 + duplicate, unexpected, successful := 0, 0, 0 defer func(start time.Time) { if duplicate > 0 || unexpected > 0 { @@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error { // Iterate over all the delivered data and inject one-by-one into the trie progress := false - for _, blob := range req.response { prog, hash, err := s.processNodeData(blob) switch err { @@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error { s.numUncommitted++ s.bytesUncommitted += len(blob) progress = progress || prog + successful++ case trie.ErrNotRequested: unexpected++ case trie.ErrAlreadyProcessed: duplicate++ default: - return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) + return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } if _, ok := req.tasks[hash]; ok { delete(req.tasks, hash) @@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error { // If we've requested the node too many times already, it may be a malicious // sync where nobody has the right data. Abort. if len(task.attempts) >= npeers { - return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) } // Missing item, place into the retry queue. s.tasks[hash] = task } - return nil + return successful, nil } // processNodeData tries to inject a trie node data blob delivered from a remote diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go new file mode 100644 index 0000000000..0b5a214257 --- /dev/null +++ b/eth/downloader/testchain_test.go @@ -0,0 +1,221 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +// Test chain parameters. +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + testDB = ethdb.NewMemDatabase() + testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000)) +) + +// The common prefix of all test chains: +var testChainBase = newTestChain(blockCacheItems+200, testGenesis) + +// Different forks on top of the base chain: +var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain + +func init() { + var forkLen = int(MaxForkAncestry + 50) + var wg sync.WaitGroup + wg.Add(3) + go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() + go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() + go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() + wg.Wait() +} + +type testChain struct { + genesis *types.Block + chain []common.Hash + headerm map[common.Hash]*types.Header + blockm map[common.Hash]*types.Block + receiptm map[common.Hash][]*types.Receipt + tdm map[common.Hash]*big.Int +} + +// newTestChain creates a blockchain of the given length. +func newTestChain(length int, genesis *types.Block) *testChain { + tc := new(testChain).copy(length) + tc.genesis = genesis + tc.chain = append(tc.chain, genesis.Hash()) + tc.headerm[tc.genesis.Hash()] = tc.genesis.Header() + tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty() + tc.blockm[tc.genesis.Hash()] = tc.genesis + tc.generate(length-1, 0, genesis, false) + return tc +} + +// makeFork creates a fork on top of the test chain. +func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { + fork := tc.copy(tc.len() + length) + fork.generate(length, seed, tc.headBlock(), heavy) + return fork +} + +// shorten creates a copy of the chain with the given length. It panics if the +// length is longer than the number of available blocks. +func (tc *testChain) shorten(length int) *testChain { + if length > tc.len() { + panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len())) + } + return tc.copy(length) +} + +func (tc *testChain) copy(newlen int) *testChain { + cpy := &testChain{ + genesis: tc.genesis, + headerm: make(map[common.Hash]*types.Header, newlen), + blockm: make(map[common.Hash]*types.Block, newlen), + receiptm: make(map[common.Hash][]*types.Receipt, newlen), + tdm: make(map[common.Hash]*big.Int, newlen), + } + for i := 0; i < len(tc.chain) && i < newlen; i++ { + hash := tc.chain[i] + cpy.chain = append(cpy.chain, tc.chain[i]) + cpy.tdm[hash] = tc.tdm[hash] + cpy.blockm[hash] = tc.blockm[hash] + cpy.headerm[hash] = tc.headerm[hash] + cpy.receiptm[hash] = tc.receiptm[hash] + } + return cpy +} + +// generate creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 22th block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { + // start := time.Now() + // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }() + + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // If a heavy chain is requested, delay blocks to raise difficulty + if heavy { + block.OffsetTime(-1) + } + // Include transactions to the miner to make blocks more interesting. + if parent == tc.genesis && i%22 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // if the block number is a multiple of 5, add a bonus uncle to the block + if i > 0 && i%5 == 0 { + block.AddUncle(&types.Header{ + ParentHash: block.PrevBlock(i - 1).Hash(), + Number: big.NewInt(block.Number().Int64() - 1), + }) + } + }) + + // Convert the block-chain into a hash-chain and header/block maps + td := new(big.Int).Set(tc.td(parent.Hash())) + for i, b := range blocks { + td := td.Add(td, b.Difficulty()) + hash := b.Hash() + tc.chain = append(tc.chain, hash) + tc.blockm[hash] = b + tc.headerm[hash] = b.Header() + tc.receiptm[hash] = receipts[i] + tc.tdm[hash] = new(big.Int).Set(td) + } +} + +// len returns the total number of blocks in the chain. +func (tc *testChain) len() int { + return len(tc.chain) +} + +// headBlock returns the head of the chain. +func (tc *testChain) headBlock() *types.Block { + return tc.blockm[tc.chain[len(tc.chain)-1]] +} + +// td returns the total difficulty of the given block. +func (tc *testChain) td(hash common.Hash) *big.Int { + return tc.tdm[hash] +} + +// headersByHash returns headers in ascending order from the given hash. +func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header { + num, _ := tc.hashToNumber(origin) + return tc.headersByNumber(num, amount, skip) +} + +// headersByNumber returns headers in ascending order from the given number. +func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header { + result := make([]*types.Header, 0, amount) + for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 { + if header, ok := tc.headerm[tc.chain[int(num)]]; ok { + result = append(result, header) + } + } + return result +} + +// receipts returns the receipts of the given block hashes. +func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt { + results := make([][]*types.Receipt, 0, len(hashes)) + for _, hash := range hashes { + if receipt, ok := tc.receiptm[hash]; ok { + results = append(results, receipt) + } + } + return results +} + +// bodies returns the block bodies of the given block hashes. +func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) { + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) + for _, hash := range hashes { + if block, ok := tc.blockm[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) + } + } + return transactions, uncles +} + +func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) { + for num, hash := range tc.chain { + if hash == target { + return uint64(num), true + } + } + return 0, false +} diff --git a/eth/gen_config.go b/eth/gen_config.go index e77effdd0b..d9ac51999d 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -28,7 +28,8 @@ func (c Config) MarshalTOML() (interface{}, error) { SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int - TrieCache int + TrieCleanCache int + TrieDirtyCache int TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` @@ -44,6 +45,8 @@ func (c Config) MarshalTOML() (interface{}, error) { EnablePreimageRecording bool DocRoot string `toml:"-"` ExitWhenSynced time.Duration + EWASMInterpreter string + EVMInterpreter string } var enc Config enc.Genesis = c.Genesis @@ -55,7 +58,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache - enc.TrieCache = c.TrieCache + enc.TrieCleanCache = c.TrieCleanCache + enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase enc.MinerNotify = c.MinerNotify @@ -71,6 +75,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EnablePreimageRecording = c.EnablePreimageRecording enc.DocRoot = c.DocRoot enc.ExitWhenSynced = c.ExitWhenSynced + enc.EWASMInterpreter = c.EWASMInterpreter + enc.EVMInterpreter = c.EVMInterpreter return &enc, nil } @@ -86,7 +92,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int - TrieCache *int + TrieCleanCache *int + TrieDirtyCache *int TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` @@ -102,6 +109,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { EnablePreimageRecording *bool DocRoot *string `toml:"-"` ExitWhenSynced *time.Duration + EWASMInterpreter *string + EVMInterpreter *string } var dec Config if err := unmarshal(&dec); err != nil { @@ -134,8 +143,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseCache != nil { c.DatabaseCache = *dec.DatabaseCache } - if dec.TrieCache != nil { - c.TrieCache = *dec.TrieCache + if dec.TrieCleanCache != nil { + c.TrieCleanCache = *dec.TrieCleanCache + } + if dec.TrieDirtyCache != nil { + c.TrieDirtyCache = *dec.TrieDirtyCache } if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout @@ -181,6 +193,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { } if dec.ExitWhenSynced != nil { c.ExitWhenSynced = *dec.ExitWhenSynced + if dec.EWASMInterpreter != nil { + c.EWASMInterpreter = *dec.EWASMInterpreter + } + if dec.EVMInterpreter != nil { + c.EVMInterpreter = *dec.EVMInterpreter } return nil } diff --git a/eth/handler.go b/eth/handler.go index 551781ef0f..741fc9d5a5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -49,6 +49,9 @@ const ( // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 + + // minimim number of peers to broadcast new blocks to + minBroadcastPeers = 4 ) var ( @@ -650,12 +653,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trueHead = request.Block.ParentHash() trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty()) ) - // Update the peers total difficulty if better than the previous + // Update the peer's total difficulty if better than the previous if _, td := p.Head(); trueTD.Cmp(td) > 0 { p.SetHead(trueHead, trueTD) // Schedule a sync if above ours. Note, this will not fire a sync for a gap of - // a singe block (as the true TD is below the propagated block), however this + // a single block (as the true TD is below the propagated block), however this // scenario should easily be covered by the fetcher. currentBlock := pm.blockchain.CurrentBlock() if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash(), currentBlock.NumberU64())) > 0 { @@ -705,7 +708,14 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { return } // Send the block to a subset of our peers - transfer := peers[:int(math.Sqrt(float64(len(peers))))] + transferLen := int(math.Sqrt(float64(len(peers)))) + if transferLen < minBroadcastPeers { + transferLen = minBroadcastPeers + } + if transferLen > len(peers) { + transferLen = len(peers) + } + transfer := peers[:transferLen] for _, peer := range transfer { peer.AsyncSendNewBlock(block, td) } diff --git a/eth/handler_test.go b/eth/handler_test.go index 0885a04487..7811cd4804 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -17,6 +17,7 @@ package eth import ( + "fmt" "math" "math/big" "math/rand" @@ -466,14 +467,17 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool } // Create a DAO aware protocol manager var ( - evmux = new(event.TypeMux) - pow = ethash.NewFaker() - db = ethdb.NewMemDatabase() - config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} - gspec = &core.Genesis{Config: config} - genesis = gspec.MustCommit(db) - blockchain, _ = core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil) + evmux = new(event.TypeMux) + pow = ethash.NewFaker() + db = ethdb.NewMemDatabase() + config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} + gspec = &core.Genesis{Config: config} + genesis = gspec.MustCommit(db) ) + blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil) + if err != nil { + t.Fatalf("failed to create new blockchain: %v", err) + } pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) @@ -520,3 +524,90 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool } } } + +func TestBroadcastBlock(t *testing.T) { + var tests = []struct { + totalPeers int + broadcastExpected int + }{ + {1, 1}, + {2, 2}, + {3, 3}, + {4, 4}, + {5, 4}, + {9, 4}, + {12, 4}, + {16, 4}, + {26, 5}, + {100, 10}, + } + for _, test := range tests { + testBroadcastBlock(t, test.totalPeers, test.broadcastExpected) + } +} + +func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { + var ( + evmux = new(event.TypeMux) + pow = ethash.NewFaker() + db = ethdb.NewMemDatabase() + config = ¶ms.ChainConfig{} + gspec = &core.Genesis{Config: config} + genesis = gspec.MustCommit(db) + ) + blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil) + if err != nil { + t.Fatalf("failed to create new blockchain: %v", err) + } + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) + if err != nil { + t.Fatalf("failed to start test protocol manager: %v", err) + } + pm.Start(1000) + defer pm.Stop() + var peers []*testPeer + for i := 0; i < totalPeers; i++ { + peer, _ := newTestPeer(fmt.Sprintf("peer %d", i), eth63, pm, true) + defer peer.close() + peers = append(peers, peer) + } + chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {}) + pm.BroadcastBlock(chain[0], true /*propagate*/) + + errCh := make(chan error, totalPeers) + doneCh := make(chan struct{}, totalPeers) + for _, peer := range peers { + go func(p *testPeer) { + if err := p2p.ExpectMsg(p.app, NewBlockMsg, &newBlockData{Block: chain[0], TD: big.NewInt(131136)}); err != nil { + errCh <- err + } else { + doneCh <- struct{}{} + } + }(peer) + } + timeoutCh := time.NewTimer(time.Millisecond * 100).C + var receivedCount int +outer: + for { + select { + case err = <-errCh: + break outer + case <-doneCh: + receivedCount++ + if receivedCount == totalPeers { + break outer + } + case <-timeoutCh: + break outer + } + } + for _, peer := range peers { + peer.app.Close() + } + if err != nil { + t.Errorf("error matching block by peer: %v", err) + } + if receivedCount != broadcastExpected { + t.Errorf("block broadcast to %d peers, expected %d", receivedCount, broadcastExpected) + } +} diff --git a/eth/protocol.go b/eth/protocol.go index 0e90e6a2ef..497ba4c597 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -37,7 +37,7 @@ const ( // ProtocolName is the official short name of the protocol used during capability negotiation. var ProtocolName = "eth" -// ProtocolVersions are the upported versions of the eth protocol (first is primary). +// ProtocolVersions are the supported versions of the eth protocol (first is primary). var ProtocolVersions = []uint{eth63, eth62} // ProtocolLengths are the number of implemented message corresponding to different protocol versions. diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index b519236f2d..3533a831f1 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -290,11 +290,12 @@ type Tracer struct { contractWrapper *contractWrapper // Wrapper around the contract object dbWrapper *dbWrapper // Wrapper around the VM environment - pcValue *uint // Swappable pc value wrapped by a log accessor - gasValue *uint // Swappable gas value wrapped by a log accessor - costValue *uint // Swappable cost value wrapped by a log accessor - depthValue *uint // Swappable depth value wrapped by a log accessor - errorValue *string // Swappable error value wrapped by a log accessor + pcValue *uint // Swappable pc value wrapped by a log accessor + gasValue *uint // Swappable gas value wrapped by a log accessor + costValue *uint // Swappable cost value wrapped by a log accessor + depthValue *uint // Swappable depth value wrapped by a log accessor + errorValue *string // Swappable error value wrapped by a log accessor + refundValue *uint // Swappable refund value wrapped by a log accessor ctx map[string]interface{} // Transaction context gathered throughout execution err error // Error, if one has occurred @@ -323,6 +324,7 @@ func New(code string) (*Tracer, error) { gasValue: new(uint), costValue: new(uint), depthValue: new(uint), + refundValue: new(uint), } // Set up builtins for this environment tracer.vm.PushGlobalGoFunction("toHex", func(ctx *duktape.Context) int { @@ -442,6 +444,9 @@ func New(code string) (*Tracer, error) { tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.depthValue); return 1 }) tracer.vm.PutPropString(logObject, "getDepth") + tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.refundValue); return 1 }) + tracer.vm.PutPropString(logObject, "getRefund") + tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { if tracer.errorValue != nil { ctx.PushString(*tracer.errorValue) @@ -527,6 +532,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost *jst.gasValue = uint(gas) *jst.costValue = uint(cost) *jst.depthValue = uint(depth) + *jst.refundValue = uint(env.StateDB.GetRefund()) jst.errorValue = nil if err != nil { diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 58b6247245..a45a121159 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" ) @@ -43,8 +44,14 @@ func (account) ReturnGas(*big.Int) {} func (account) SetCode(common.Hash, []byte) {} func (account) ForEachStorage(cb func(key, value common.Hash) bool) {} +type dummyStatedb struct { + state.StateDB +} + +func (*dummyStatedb) GetRefund() uint64 { return 1337 } + func runTrace(tracer *Tracer) (json.RawMessage, error) { - env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) + env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} @@ -126,7 +133,7 @@ func TestHaltBetweenSteps(t *testing.T) { t.Fatal(err) } - env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) + env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index b40837c8cc..f3163e19b3 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb // FilterLogs executes a filter query. func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { var result []types.Log - err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q)) + arg, err := toFilterArg(q) + if err != nil { + return nil, err + } + err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg) return result, err } // SubscribeFilterLogs subscribes to the results of a streaming filter query. func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q)) + arg, err := toFilterArg(q) + if err != nil { + return nil, err + } + return ec.c.EthSubscribe(ctx, ch, "logs", arg) } -func toFilterArg(q ethereum.FilterQuery) interface{} { +func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { arg := map[string]interface{}{ - "fromBlock": toBlockNumArg(q.FromBlock), - "toBlock": toBlockNumArg(q.ToBlock), - "address": q.Addresses, - "topics": q.Topics, + "address": q.Addresses, + "topics": q.Topics, } - if q.FromBlock == nil { - arg["fromBlock"] = "0x0" + if q.BlockHash != nil { + arg["blockHash"] = *q.BlockHash + if q.FromBlock != nil || q.ToBlock != nil { + return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + } + } else { + if q.FromBlock == nil { + arg["fromBlock"] = "0x0" + } else { + arg["fromBlock"] = toBlockNumArg(q.FromBlock) + } + arg["toBlock"] = toBlockNumArg(q.ToBlock) } - return arg + return arg, nil } // Pending State diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 178eb2be98..3e8bf974c2 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -16,7 +16,15 @@ package ethclient -import "github.com/ethereum/go-ethereum" +import ( + "fmt" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" +) // Verify that Client implements the ethereum interfaces. var ( @@ -32,3 +40,113 @@ var ( // _ = ethereum.PendingStateEventer(&Client{}) _ = ethereum.PendingContractCaller(&Client{}) ) + +func TestToFilterArg(t *testing.T) { + blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock") + addresses := []common.Address{ + common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"), + } + blockHash := common.HexToHash( + "0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb", + ) + + for _, testCase := range []struct { + name string + input ethereum.FilterQuery + output interface{} + err error + }{ + { + "without BlockHash", + ethereum.FilterQuery{ + Addresses: addresses, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "fromBlock": "0x1", + "toBlock": "0x2", + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with nil fromBlock and nil toBlock", + ethereum.FilterQuery{ + Addresses: addresses, + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "fromBlock": "0x0", + "toBlock": "latest", + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with blockhash", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + Topics: [][]common.Hash{}, + }, + map[string]interface{}{ + "address": addresses, + "blockHash": blockHash, + "topics": [][]common.Hash{}, + }, + nil, + }, + { + "with blockhash and from block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + FromBlock: big.NewInt(1), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + { + "with blockhash and to block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + ToBlock: big.NewInt(1), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + { + "with blockhash and both from / to block", + ethereum.FilterQuery{ + Addresses: addresses, + BlockHash: &blockHash, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + Topics: [][]common.Hash{}, + }, + nil, + blockHashErr, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + output, err := toFilterArg(testCase.input) + if (testCase.err == nil) != (err == nil) { + t.Fatalf("expected error %v but got %v", testCase.err, err) + } + if testCase.err != nil { + if testCase.err.Error() != err.Error() { + t.Fatalf("expected error %v but got %v", testCase.err, err) + } + } else if !reflect.DeepEqual(testCase.output, output) { + t.Fatalf("expected filter arg %v but got %v", testCase.output, output) + } + }) + } +} diff --git a/ethdb/database.go b/ethdb/database.go index 99abd09b99..6c62d6a386 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// +build !js + package ethdb import ( @@ -380,71 +382,3 @@ func (b *ldbBatch) Reset() { b.b.Reset() b.size = 0 } - -type table struct { - db Database - prefix string -} - -// NewTable returns a Database object that prefixes all keys with a given -// string. -func NewTable(db Database, prefix string) Database { - return &table{ - db: db, - prefix: prefix, - } -} - -func (dt *table) Put(key []byte, value []byte) error { - return dt.db.Put(append([]byte(dt.prefix), key...), value) -} - -func (dt *table) Has(key []byte) (bool, error) { - return dt.db.Has(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Get(key []byte) ([]byte, error) { - return dt.db.Get(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Delete(key []byte) error { - return dt.db.Delete(append([]byte(dt.prefix), key...)) -} - -func (dt *table) Close() { - // Do nothing; don't close the underlying DB. -} - -type tableBatch struct { - batch Batch - prefix string -} - -// NewTableBatch returns a Batch object which prefixes all keys with a given string. -func NewTableBatch(db Database, prefix string) Batch { - return &tableBatch{db.NewBatch(), prefix} -} - -func (dt *table) NewBatch() Batch { - return &tableBatch{dt.db.NewBatch(), dt.prefix} -} - -func (tb *tableBatch) Put(key, value []byte) error { - return tb.batch.Put(append([]byte(tb.prefix), key...), value) -} - -func (tb *tableBatch) Delete(key []byte) error { - return tb.batch.Delete(append([]byte(tb.prefix), key...)) -} - -func (tb *tableBatch) Write() error { - return tb.batch.Write() -} - -func (tb *tableBatch) ValueSize() int { - return tb.batch.ValueSize() -} - -func (tb *tableBatch) Reset() { - tb.batch.Reset() -} diff --git a/ethdb/database_js.go b/ethdb/database_js.go new file mode 100644 index 0000000000..ba6eeb5a23 --- /dev/null +++ b/ethdb/database_js.go @@ -0,0 +1,68 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build js + +package ethdb + +import ( + "errors" +) + +var errNotSupported = errors.New("ethdb: not supported") + +type LDBDatabase struct { +} + +// NewLDBDatabase returns a LevelDB wrapped object. +func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { + return nil, errNotSupported +} + +// Path returns the path to the database directory. +func (db *LDBDatabase) Path() string { + return "" +} + +// Put puts the given key / value to the queue +func (db *LDBDatabase) Put(key []byte, value []byte) error { + return errNotSupported +} + +func (db *LDBDatabase) Has(key []byte) (bool, error) { + return false, errNotSupported +} + +// Get returns the given key if it's present. +func (db *LDBDatabase) Get(key []byte) ([]byte, error) { + return nil, errNotSupported +} + +// Delete deletes the key from the queue and database +func (db *LDBDatabase) Delete(key []byte) error { + return errNotSupported +} + +func (db *LDBDatabase) Close() { +} + +// Meter configures the database metrics collectors and +func (db *LDBDatabase) Meter(prefix string) { +} + +func (db *LDBDatabase) NewBatch() Batch { + return nil +} diff --git a/event/filter/generic_filter.go b/ethdb/database_js_test.go similarity index 55% rename from event/filter/generic_filter.go rename to ethdb/database_js_test.go index d679b8bfa8..b4c12ae0bc 100644 --- a/event/filter/generic_filter.go +++ b/ethdb/database_js_test.go @@ -14,35 +14,12 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package filter +// +build js -type Generic struct { - Str1, Str2, Str3 string - Data map[string]struct{} +package ethdb_test - Fn func(data interface{}) -} +import ( + "github.com/ethereum/go-ethereum/ethdb" +) -// self = registered, f = incoming -func (self Generic) Compare(f Filter) bool { - var strMatch, dataMatch = true, true - - filter := f.(Generic) - if (len(self.Str1) > 0 && filter.Str1 != self.Str1) || - (len(self.Str2) > 0 && filter.Str2 != self.Str2) || - (len(self.Str3) > 0 && filter.Str3 != self.Str3) { - strMatch = false - } - - for k := range self.Data { - if _, ok := filter.Data[k]; !ok { - return false - } - } - - return strMatch && dataMatch -} - -func (self Generic) Trigger(data interface{}) { - self.Fn(data) -} +var _ ethdb.Database = ðdb.LDBDatabase{} diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 74675cbe63..382fedbf9e 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// +build !js + package ethdb_test import ( diff --git a/ethdb/table.go b/ethdb/table.go new file mode 100644 index 0000000000..28069c078e --- /dev/null +++ b/ethdb/table.go @@ -0,0 +1,51 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethdb + +type table struct { + db Database + prefix string +} + +// NewTable returns a Database object that prefixes all keys with a given +// string. +func NewTable(db Database, prefix string) Database { + return &table{ + db: db, + prefix: prefix, + } +} + +func (dt *table) Put(key []byte, value []byte) error { + return dt.db.Put(append([]byte(dt.prefix), key...), value) +} + +func (dt *table) Has(key []byte) (bool, error) { + return dt.db.Has(append([]byte(dt.prefix), key...)) +} + +func (dt *table) Get(key []byte) ([]byte, error) { + return dt.db.Get(append([]byte(dt.prefix), key...)) +} + +func (dt *table) Delete(key []byte) error { + return dt.db.Delete(append([]byte(dt.prefix), key...)) +} + +func (dt *table) Close() { + // Do nothing; don't close the underlying DB. +} diff --git a/ethdb/table_batch.go b/ethdb/table_batch.go new file mode 100644 index 0000000000..ae83e79ced --- /dev/null +++ b/ethdb/table_batch.go @@ -0,0 +1,51 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethdb + +type tableBatch struct { + batch Batch + prefix string +} + +// NewTableBatch returns a Batch object which prefixes all keys with a given string. +func NewTableBatch(db Database, prefix string) Batch { + return &tableBatch{db.NewBatch(), prefix} +} + +func (dt *table) NewBatch() Batch { + return &tableBatch{dt.db.NewBatch(), dt.prefix} +} + +func (tb *tableBatch) Put(key, value []byte) error { + return tb.batch.Put(append([]byte(tb.prefix), key...), value) +} + +func (tb *tableBatch) Delete(key []byte) error { + return tb.batch.Delete(append([]byte(tb.prefix), key...)) +} + +func (tb *tableBatch) Write() error { + return tb.batch.Write() +} + +func (tb *tableBatch) ValueSize() int { + return tb.batch.ValueSize() +} + +func (tb *tableBatch) Reset() { + tb.batch.Reset() +} diff --git a/event/event_test.go b/event/event_test.go index a12945a471..2be357ba2d 100644 --- a/event/event_test.go +++ b/event/event_test.go @@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) { } } -func emptySubscriber(mux *TypeMux, types ...interface{}) { +func emptySubscriber(mux *TypeMux) { s := mux.Subscribe(testEvent(0)) go func() { for range s.Chan() { @@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) { func BenchmarkPostConcurrent(b *testing.B) { var mux = new(TypeMux) defer mux.Stop() - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) + emptySubscriber(mux) + emptySubscriber(mux) + emptySubscriber(mux) var wg sync.WaitGroup poster := func() { diff --git a/event/filter/filter.go b/event/filter/filter.go deleted file mode 100644 index a6fe46d6a0..0000000000 --- a/event/filter/filter.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package filter implements event filters. -package filter - -import "reflect" - -type Filter interface { - Compare(Filter) bool - Trigger(data interface{}) -} - -type FilterEvent struct { - filter Filter - data interface{} -} - -type Filters struct { - id int - watchers map[int]Filter - ch chan FilterEvent - - quit chan struct{} -} - -func New() *Filters { - return &Filters{ - ch: make(chan FilterEvent), - watchers: make(map[int]Filter), - quit: make(chan struct{}), - } -} - -func (f *Filters) Start() { - go f.loop() -} - -func (f *Filters) Stop() { - close(f.quit) -} - -func (f *Filters) Notify(filter Filter, data interface{}) { - f.ch <- FilterEvent{filter, data} -} - -func (f *Filters) Install(watcher Filter) int { - f.watchers[f.id] = watcher - f.id++ - - return f.id - 1 -} - -func (f *Filters) Uninstall(id int) { - delete(f.watchers, id) -} - -func (f *Filters) loop() { -out: - for { - select { - case <-f.quit: - break out - case event := <-f.ch: - for _, watcher := range f.watchers { - if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) { - if watcher.Compare(event.filter) { - watcher.Trigger(event.data) - } - } - } - } - } -} - -func (f *Filters) Match(a, b Filter) bool { - return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b) -} - -func (f *Filters) Get(i int) Filter { - return f.watchers[i] -} diff --git a/event/filter/filter_test.go b/event/filter/filter_test.go deleted file mode 100644 index dcc9112453..0000000000 --- a/event/filter/filter_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package filter - -import ( - "testing" - "time" -) - -// Simple test to check if baseline matching/mismatching filtering works. -func TestFilters(t *testing.T) { - fm := New() - fm.Start() - - // Register two filters to catch posted data - first := make(chan struct{}) - fm.Install(Generic{ - Str1: "hello", - Fn: func(data interface{}) { - first <- struct{}{} - }, - }) - second := make(chan struct{}) - fm.Install(Generic{ - Str1: "hello1", - Str2: "hello", - Fn: func(data interface{}) { - second <- struct{}{} - }, - }) - // Post an event that should only match the first filter - fm.Notify(Generic{Str1: "hello"}, true) - fm.Stop() - - // Ensure only the mathcing filters fire - select { - case <-first: - case <-time.After(100 * time.Millisecond): - t.Error("matching filter timed out") - } - select { - case <-second: - t.Error("mismatching filter fired") - case <-time.After(100 * time.Millisecond): - } -} diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 7d7eba98a7..46c8fe9f80 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -25,11 +25,11 @@ import ( "runtime" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/log/term" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics/exp" "github.com/fjl/memsize/memsizeui" colorable "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" "gopkg.in/urfave/cli.v1" ) @@ -101,7 +101,7 @@ var ( ) func init() { - usecolor := term.IsTty(os.Stderr.Fd()) && os.Getenv("TERM") != "dumb" + usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" output := io.Writer(os.Stderr) if usecolor { output = colorable.NewColorableStderr() diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 8016ebe3d7..43a33e9921 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -328,6 +328,9 @@ func (s *PrivateAccountAPI) UnlockAccount(addr common.Address, password string, d = time.Duration(*duration) * time.Second } err := fetchKeystore(s.am).TimedUnlock(accounts.Account{Address: addr}, password, d) + if err != nil { + log.Warn("Failed account unlock attempt", "address", addr, "err", err) + } return err == nil, err } @@ -336,10 +339,10 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { return fetchKeystore(s.am).Lock(addr) == nil } -// signTransactions sets defaults and signs the given transaction +// signTransaction sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool -func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args SendTxArgs, passwd string) (*types.Transaction, error) { +func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: args.From} wallet, err := s.am.Find(account) @@ -370,8 +373,9 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs s.nonceLock.LockAddr(args.From) defer s.nonceLock.UnlockAddr(args.From) } - signed, err := s.signTransaction(ctx, args, passwd) + signed, err := s.signTransaction(ctx, &args, passwd) if err != nil { + log.Warn("Failed transaction send attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err) return common.Hash{}, err } return submitTransaction(ctx, s.b, signed) @@ -393,8 +397,9 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs if args.Nonce == nil { return nil, fmt.Errorf("nonce not specified") } - signed, err := s.signTransaction(ctx, args, passwd) + signed, err := s.signTransaction(ctx, &args, passwd) if err != nil { + log.Warn("Failed transaction sign attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err) return nil, err } data, err := rlp.EncodeToBytes(signed) @@ -436,6 +441,7 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr c // Assemble sign the data with the wallet signature, err := wallet.SignHashWithPassphrase(account, passwd, signHash(data)) if err != nil { + log.Warn("Failed data sign attempt", "address", addr, "err", err) return nil, err } signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper @@ -502,6 +508,72 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add return (*hexutil.Big)(state.GetBalance(address)), state.Error() } +// Result structs for GetProof +type AccountResult struct { + Address common.Address `json:"address"` + AccountProof []string `json:"accountProof"` + Balance *hexutil.Big `json:"balance"` + CodeHash common.Hash `json:"codeHash"` + Nonce hexutil.Uint64 `json:"nonce"` + StorageHash common.Hash `json:"storageHash"` + StorageProof []StorageResult `json:"storageProof"` +} +type StorageResult struct { + Key string `json:"key"` + Value *hexutil.Big `json:"value"` + Proof []string `json:"proof"` +} + +// GetProof returns the Merkle-proof for a given account and optionally some storage keys. +func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNr rpc.BlockNumber) (*AccountResult, error) { + state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) + if state == nil || err != nil { + return nil, err + } + + storageTrie := state.StorageTrie(address) + storageHash := types.EmptyRootHash + codeHash := state.GetCodeHash(address) + storageProof := make([]StorageResult, len(storageKeys)) + + // if we have a storageTrie, (which means the account exists), we can update the storagehash + if storageTrie != nil { + storageHash = storageTrie.Hash() + } else { + // no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray. + codeHash = crypto.Keccak256Hash(nil) + } + + // create the proof for the storageKeys + for i, key := range storageKeys { + if storageTrie != nil { + proof, storageError := state.GetStorageProof(address, common.HexToHash(key)) + if storageError != nil { + return nil, storageError + } + storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)} + } else { + storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}} + } + } + + // create the accountProof + accountProof, proofErr := state.GetProof(address) + if proofErr != nil { + return nil, proofErr + } + + return &AccountResult{ + Address: address, + AccountProof: common.ToHexArray(accountProof), + Balance: (*hexutil.Big)(state.GetBalance(address)), + CodeHash: codeHash, + Nonce: hexutil.Uint64(state.GetNonce(address)), + StorageHash: storageHash, + StorageProof: storageProof, + }, state.Error() +} + // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all // transactions in the block are returned in full detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index f4eb47a12a..a5f3196535 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -378,6 +378,12 @@ web3._extend({ params: 2, inputFormatter: [null, null] }), + new web3._extend.Method({ + name: 'traceBadBlock', + call: 'debug_traceBadBlock', + params: 1, + inputFormatter: [null] + }), new web3._extend.Method({ name: 'traceBlockByNumber', call: 'debug_traceBlockByNumber', @@ -433,6 +439,11 @@ const Eth_JS = ` web3._extend({ property: 'eth', methods: [ + new web3._extend.Method({ + name: 'chainId', + call: 'eth_chainId', + params: 0 + }), new web3._extend.Method({ name: 'sign', call: 'eth_sign', @@ -470,6 +481,12 @@ web3._extend({ params: 2, inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex] }), + new web3._extend.Method({ + name: 'getProof', + call: 'eth_getProof', + params: 3, + inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter] + }), ], properties: [ new web3._extend.Property({ diff --git a/les/distributor_test.go b/les/distributor_test.go index 2891bcab49..8c7621f26b 100644 --- a/les/distributor_test.go +++ b/les/distributor_test.go @@ -87,7 +87,7 @@ const ( testDistBufLimit = 10000000 testDistMaxCost = 1000000 testDistPeerCount = 5 - testDistReqCount = 50000 + testDistReqCount = 5000 testDistMaxResendCount = 3 ) diff --git a/les/fetcher.go b/les/fetcher.go index cc539c42bf..f0d3b188db 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -32,8 +32,9 @@ import ( ) const ( - blockDelayTimeout = time.Second * 10 // timeout for a peer to announce a head that has already been confirmed by others - maxNodeCount = 20 // maximum number of fetcherTreeNode entries remembered for each peer + blockDelayTimeout = time.Second * 10 // timeout for a peer to announce a head that has already been confirmed by others + maxNodeCount = 20 // maximum number of fetcherTreeNode entries remembered for each peer + serverStateAvailable = 100 // number of recent blocks where state availability is assumed ) // lightFetcher implements retrieval of newly announced headers. It also provides a peerHasBlock function for the @@ -215,8 +216,8 @@ func (f *lightFetcher) syncLoop() { // registerPeer adds a new peer to the fetcher's peer set func (f *lightFetcher) registerPeer(p *peer) { p.lock.Lock() - p.hasBlock = func(hash common.Hash, number uint64) bool { - return f.peerHasBlock(p, hash, number) + p.hasBlock = func(hash common.Hash, number uint64, hasState bool) bool { + return f.peerHasBlock(p, hash, number, hasState) } p.lock.Unlock() @@ -344,21 +345,27 @@ func (f *lightFetcher) announce(p *peer, head *announceData) { // peerHasBlock returns true if we can assume the peer knows the given block // based on its announcements -func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bool { +func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64, hasState bool) bool { f.lock.Lock() defer f.lock.Unlock() + fp := f.peers[p] + if fp == nil || fp.root == nil { + return false + } + + if hasState { + if fp.lastAnnounced == nil || fp.lastAnnounced.number > number+serverStateAvailable { + return false + } + } + if f.syncing { // always return true when syncing // false positives are acceptable, a more sophisticated condition can be implemented later return true } - fp := f.peers[p] - if fp == nil || fp.root == nil { - return false - } - if number >= fp.root.number { // it is recent enough that if it is known, is should be in the peer's block tree return fp.nodeByHash[hash] != nil diff --git a/les/odr_requests.go b/les/odr_requests.go index 77b1b6d0c8..0f2e5dd9ec 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -84,7 +84,7 @@ func (r *BlockRequest) GetCost(peer *peer) uint64 { // CanSend tells if a certain peer is suitable for serving the given request func (r *BlockRequest) CanSend(peer *peer) bool { - return peer.HasBlock(r.Hash, r.Number) + return peer.HasBlock(r.Hash, r.Number, false) } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -140,7 +140,7 @@ func (r *ReceiptsRequest) GetCost(peer *peer) uint64 { // CanSend tells if a certain peer is suitable for serving the given request func (r *ReceiptsRequest) CanSend(peer *peer) bool { - return peer.HasBlock(r.Hash, r.Number) + return peer.HasBlock(r.Hash, r.Number, false) } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -202,7 +202,7 @@ func (r *TrieRequest) GetCost(peer *peer) uint64 { // CanSend tells if a certain peer is suitable for serving the given request func (r *TrieRequest) CanSend(peer *peer) bool { - return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber) + return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber, true) } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -272,7 +272,7 @@ func (r *CodeRequest) GetCost(peer *peer) uint64 { // CanSend tells if a certain peer is suitable for serving the given request func (r *CodeRequest) CanSend(peer *peer) bool { - return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber) + return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber, true) } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) diff --git a/les/odr_test.go b/les/odr_test.go index e6458adf56..ac81fbcf02 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -194,7 +194,7 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { client.peers.Register(client.rPeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed client.peers.lock.Lock() - client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true } + client.rPeer.hasBlock = func(common.Hash, uint64, bool) bool { return true } client.peers.lock.Unlock() test(5) // still expect all retrievals to pass, now data should be cached locally diff --git a/les/peer.go b/les/peer.go index 1f343847e9..678384f0eb 100644 --- a/les/peer.go +++ b/les/peer.go @@ -67,7 +67,7 @@ type peer struct { sendQueue *execQueue poolEntry *poolEntry - hasBlock func(common.Hash, uint64) bool + hasBlock func(common.Hash, uint64, bool) bool responseErrors int fcClient *flowcontrol.ClientNode // nil if the peer is server only @@ -171,11 +171,11 @@ func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 { } // HasBlock checks if the peer has a given block -func (p *peer) HasBlock(hash common.Hash, number uint64) bool { +func (p *peer) HasBlock(hash common.Hash, number uint64, hasState bool) bool { p.lock.RLock() hasBlock := p.hasBlock p.lock.RUnlock() - return hasBlock != nil && hasBlock(hash, number) + return hasBlock != nil && hasBlock(hash, number, hasState) } // SendAnnounce announces the availability of a number of blocks through diff --git a/les/request_test.go b/les/request_test.go index f02c2a3d76..c9c1851982 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -115,7 +115,7 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { client.peers.Register(client.rPeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed client.rPeer.lock.Lock() - client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true } + client.rPeer.hasBlock = func(common.Hash, uint64, bool) bool { return true } client.rPeer.lock.Unlock() // expect all retrievals to pass test(5) diff --git a/les/serverpool.go b/les/serverpool.go index 0fe6e49b61..52b54b371f 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -683,7 +683,7 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error { } func encodePubkey64(pub *ecdsa.PublicKey) []byte { - return crypto.FromECDSAPub(pub)[:1] + return crypto.FromECDSAPub(pub)[1:] } func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) { diff --git a/light/postprocess.go b/light/postprocess.go index 2f8cb73ab1..dd1b74a7be 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -84,23 +84,23 @@ var ( } // TestServerIndexerConfig wraps a set of configs as a test indexer config for server side. TestServerIndexerConfig = &IndexerConfig{ - ChtSize: 256, - PairChtSize: 2048, - ChtConfirms: 16, - BloomSize: 256, - BloomConfirms: 16, - BloomTrieSize: 2048, - BloomTrieConfirms: 16, + ChtSize: 64, + PairChtSize: 512, + ChtConfirms: 4, + BloomSize: 64, + BloomConfirms: 4, + BloomTrieSize: 512, + BloomTrieConfirms: 4, } // TestClientIndexerConfig wraps a set of configs as a test indexer config for client side. TestClientIndexerConfig = &IndexerConfig{ - ChtSize: 2048, - PairChtSize: 256, - ChtConfirms: 128, - BloomSize: 2048, - BloomConfirms: 128, - BloomTrieSize: 2048, - BloomTrieConfirms: 128, + ChtSize: 512, + PairChtSize: 64, + ChtConfirms: 32, + BloomSize: 512, + BloomConfirms: 32, + BloomTrieSize: 512, + BloomTrieConfirms: 32, } ) @@ -159,7 +159,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *co diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down sectionSize: size, } return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") @@ -281,7 +281,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + triedb: trie.NewDatabaseWithCache(trieTable, 1), // Use a tiny cache only to keep memory down parentSize: parentSize, size: size, } diff --git a/log/term/LICENSE b/log/term/LICENSE deleted file mode 100644 index f090cb42f3..0000000000 --- a/log/term/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Simon Eskildsen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/log/term/terminal_appengine.go b/log/term/terminal_appengine.go deleted file mode 100644 index c1b5d2a3b1..0000000000 --- a/log/term/terminal_appengine.go +++ /dev/null @@ -1,13 +0,0 @@ -// Based on ssh/terminal: -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build appengine - -package term - -// IsTty always returns false on AppEngine. -func IsTty(fd uintptr) bool { - return false -} diff --git a/log/term/terminal_darwin.go b/log/term/terminal_darwin.go deleted file mode 100644 index d8f351b1b1..0000000000 --- a/log/term/terminal_darwin.go +++ /dev/null @@ -1,13 +0,0 @@ -// Based on ssh/terminal: -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// +build !appengine - -package term - -import "syscall" - -const ioctlReadTermios = syscall.TIOCGETA - -type Termios syscall.Termios diff --git a/log/term/terminal_freebsd.go b/log/term/terminal_freebsd.go deleted file mode 100644 index cfaceab337..0000000000 --- a/log/term/terminal_freebsd.go +++ /dev/null @@ -1,18 +0,0 @@ -package term - -import ( - "syscall" -) - -const ioctlReadTermios = syscall.TIOCGETA - -// Go 1.2 doesn't include Termios for FreeBSD. This should be added in 1.3 and this could be merged with terminal_darwin. -type Termios struct { - Iflag uint32 - Oflag uint32 - Cflag uint32 - Lflag uint32 - Cc [20]uint8 - Ispeed uint32 - Ospeed uint32 -} diff --git a/log/term/terminal_linux.go b/log/term/terminal_linux.go deleted file mode 100644 index 5290468d69..0000000000 --- a/log/term/terminal_linux.go +++ /dev/null @@ -1,14 +0,0 @@ -// Based on ssh/terminal: -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine - -package term - -import "syscall" - -const ioctlReadTermios = syscall.TCGETS - -type Termios syscall.Termios diff --git a/log/term/terminal_netbsd.go b/log/term/terminal_netbsd.go deleted file mode 100644 index f9bb9e1c23..0000000000 --- a/log/term/terminal_netbsd.go +++ /dev/null @@ -1,7 +0,0 @@ -package term - -import "syscall" - -const ioctlReadTermios = syscall.TIOCGETA - -type Termios syscall.Termios diff --git a/log/term/terminal_notwindows.go b/log/term/terminal_notwindows.go deleted file mode 100644 index c9af534f62..0000000000 --- a/log/term/terminal_notwindows.go +++ /dev/null @@ -1,20 +0,0 @@ -// Based on ssh/terminal: -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux,!appengine darwin freebsd openbsd netbsd - -package term - -import ( - "syscall" - "unsafe" -) - -// IsTty returns true if the given file descriptor is a terminal. -func IsTty(fd uintptr) bool { - var termios Termios - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) - return err == 0 -} diff --git a/log/term/terminal_openbsd.go b/log/term/terminal_openbsd.go deleted file mode 100644 index f9bb9e1c23..0000000000 --- a/log/term/terminal_openbsd.go +++ /dev/null @@ -1,7 +0,0 @@ -package term - -import "syscall" - -const ioctlReadTermios = syscall.TIOCGETA - -type Termios syscall.Termios diff --git a/log/term/terminal_solaris.go b/log/term/terminal_solaris.go deleted file mode 100644 index 033c163246..0000000000 --- a/log/term/terminal_solaris.go +++ /dev/null @@ -1,9 +0,0 @@ -package term - -import "golang.org/x/sys/unix" - -// IsTty returns true if the given file descriptor is a terminal. -func IsTty(fd uintptr) bool { - _, err := unix.IoctlGetTermios(int(fd), unix.TCGETA) - return err == nil -} diff --git a/log/term/terminal_windows.go b/log/term/terminal_windows.go deleted file mode 100644 index df3c30c158..0000000000 --- a/log/term/terminal_windows.go +++ /dev/null @@ -1,26 +0,0 @@ -// Based on ssh/terminal: -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package term - -import ( - "syscall" - "unsafe" -) - -var kernel32 = syscall.NewLazyDLL("kernel32.dll") - -var ( - procGetConsoleMode = kernel32.NewProc("GetConsoleMode") -) - -// IsTty returns true if the given file descriptor is a terminal. -func IsTty(fd uintptr) bool { - var st uint32 - r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0) - return r != 0 && e == 0 -} diff --git a/metrics/counter.go b/metrics/counter.go index c7f2b4bd3a..2f78c90d5c 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -1,6 +1,8 @@ package metrics -import "sync/atomic" +import ( + "sync/atomic" +) // Counters hold an int64 value that can be incremented and decremented. type Counter interface { @@ -20,6 +22,17 @@ func GetOrRegisterCounter(name string, r Registry) Counter { return r.GetOrRegister(name, NewCounter).(Counter) } +// GetOrRegisterCounterForced returns an existing Counter or constructs and registers a +// new Counter no matter the global switch is enabled or not. +// Be sure to unregister the counter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterCounterForced(name string, r Registry) Counter { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewCounterForced).(Counter) +} + // NewCounter constructs a new StandardCounter. func NewCounter() Counter { if !Enabled { @@ -28,6 +41,12 @@ func NewCounter() Counter { return &StandardCounter{0} } +// NewCounterForced constructs a new StandardCounter and returns it no matter if +// the global switch is enabled or not. +func NewCounterForced() Counter { + return &StandardCounter{0} +} + // NewRegisteredCounter constructs and registers a new StandardCounter. func NewRegisteredCounter(name string, r Registry) Counter { c := NewCounter() @@ -38,6 +57,19 @@ func NewRegisteredCounter(name string, r Registry) Counter { return c } +// NewRegisteredCounterForced constructs and registers a new StandardCounter +// and launches a goroutine no matter the global switch is enabled or not. +// Be sure to unregister the counter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredCounterForced(name string, r Registry) Counter { + c := NewCounterForced() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + // CounterSnapshot is a read-only copy of another Counter. type CounterSnapshot int64 diff --git a/metrics/registry.go b/metrics/registry.go index cc34c9dfd2..c1cf7906ce 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() { r.underlying.UnregisterAll() } -var DefaultRegistry Registry = NewRegistry() +var ( + DefaultRegistry = NewRegistry() + EphemeralRegistry = NewRegistry() +) // Call the given function for each registered metric. func Each(f func(string, interface{})) { diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 8961091d56..7e19975aec 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -22,7 +22,6 @@ package main import ( "bytes" "crypto/ecdsa" - "fmt" "io/ioutil" "math/big" "math/rand" @@ -40,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,11 +61,11 @@ func main() { var ( nodes []*node.Node - enodes []string + enodes []*enode.Node ) for _, sealer := range sealers { // Start the node and wait until it's up - node, err := makeSealer(genesis, enodes) + node, err := makeSealer(genesis) if err != nil { panic(err) } @@ -76,18 +75,12 @@ func main() { time.Sleep(250 * time.Millisecond) } // Connect the node to al the previous ones - for _, enode := range enodes { - enode, err := discover.ParseNode(enode) - if err != nil { - panic(err) - } - node.Server().AddPeer(enode) + for _, n := range enodes { + node.Server().AddPeer(n) } - // Start tracking the node and it's enode url + // Start tracking the node and it's enode nodes = append(nodes, node) - - enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) - enodes = append(enodes, enode) + enodes = append(enodes, node.Server().Self()) // Inject the signer key and start sealing with it store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core return genesis } -func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { +func makeSealer(genesis *core.Genesis) (*node.Node, error) { // Define the basic configurations for the Ethereum node datadir, _ := ioutil.TempDir("", "") diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 5ed11d73a5..044ca9a218 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -21,7 +21,6 @@ package main import ( "crypto/ecdsa" - "fmt" "io/ioutil" "math/big" "math/rand" @@ -41,7 +40,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" ) @@ -62,11 +61,11 @@ func main() { var ( nodes []*node.Node - enodes []string + enodes []*enode.Node ) for i := 0; i < 4; i++ { // Start the node and wait until it's up - node, err := makeMiner(genesis, enodes) + node, err := makeMiner(genesis) if err != nil { panic(err) } @@ -76,18 +75,12 @@ func main() { time.Sleep(250 * time.Millisecond) } // Connect the node to al the previous ones - for _, enode := range enodes { - enode, err := discover.ParseNode(enode) - if err != nil { - panic(err) - } - node.Server().AddPeer(enode) + for _, n := range enodes { + node.Server().AddPeer(n) } - // Start tracking the node and it's enode url + // Start tracking the node and it's enode nodes = append(nodes, node) - - enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) - enodes = append(enodes, enode) + enodes = append(enodes, node.Server().Self()) // Inject the signer key and start sealing with it store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { return genesis } -func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { +func makeMiner(genesis *core.Genesis) (*node.Node, error) { // Define the basic configurations for the Ethereum node datadir, _ := ioutil.TempDir("", "") diff --git a/miner/worker_test.go b/miner/worker_test.go index db0ff4340f..7c8f167a12 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -390,12 +390,12 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co case 1: origin := float64(3 * time.Second.Nanoseconds()) estimate := origin*(1-intervalAdjustRatio) + intervalAdjustRatio*(origin/0.8+intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(int(estimate))*time.Nanosecond + wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond case 2: estimate := result[index-1] min := float64(3 * time.Second.Nanoseconds()) estimate = estimate*(1-intervalAdjustRatio) + intervalAdjustRatio*(min-intervalAdjustBias) - wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(int(estimate))*time.Nanosecond + wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(estimate)*time.Nanosecond case 3: wantMinInterval, wantRecommitInterval = time.Second, time.Second } diff --git a/node/node.go b/node/node.go index ada3837217..85299dba74 100644 --- a/node/node.go +++ b/node/node.go @@ -549,11 +549,23 @@ func (n *Node) IPCEndpoint() string { // HTTPEndpoint retrieves the current HTTP endpoint used by the protocol stack. func (n *Node) HTTPEndpoint() string { + n.lock.Lock() + defer n.lock.Unlock() + + if n.httpListener != nil { + return n.httpListener.Addr().String() + } return n.httpEndpoint } // WSEndpoint retrieves the current WS endpoint used by the protocol stack. func (n *Node) WSEndpoint() string { + n.lock.Lock() + defer n.lock.Unlock() + + if n.wsListener != nil { + return n.wsListener.Addr().String() + } return n.wsEndpoint } diff --git a/node/node_test.go b/node/node_test.go index e51900bd1e..f833cd6880 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -454,9 +454,9 @@ func TestProtocolGather(t *testing.T) { Count int Maker InstrumentingWrapper }{ - "Zero Protocols": {0, InstrumentedServiceMakerA}, - "Single Protocol": {1, InstrumentedServiceMakerB}, - "Many Protocols": {25, InstrumentedServiceMakerC}, + "zero": {0, InstrumentedServiceMakerA}, + "one": {1, InstrumentedServiceMakerB}, + "many": {10, InstrumentedServiceMakerC}, } for id, config := range services { protocols := make([]p2p.Protocol, config.Count) @@ -480,7 +480,7 @@ func TestProtocolGather(t *testing.T) { defer stack.Stop() protocols := stack.Server().Protocols - if len(protocols) != 26 { + if len(protocols) != 11 { t.Fatalf("mismatching number of protocols launched: have %d, want %d", len(protocols), 26) } for id, config := range services { diff --git a/p2p/dial.go b/p2p/dial.go index 359cdbcbba..075a0f9368 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -71,6 +71,7 @@ type dialstate struct { maxDynDials int ntab discoverTable netrestrict *netutil.Netlist + self enode.ID lookupRunning bool dialing map[enode.ID]connFlag @@ -84,7 +85,6 @@ type dialstate struct { } type discoverTable interface { - Self() *enode.Node Close() Resolve(*enode.Node) *enode.Node LookupRandom() []*enode.Node @@ -126,10 +126,11 @@ type waitExpireTask struct { time.Duration } -func newDialState(static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate { +func newDialState(self enode.ID, static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate { s := &dialstate{ maxDynDials: maxdyn, ntab: ntab, + self: self, netrestrict: netrestrict, static: make(map[enode.ID]*dialTask), dialing: make(map[enode.ID]connFlag), @@ -266,7 +267,7 @@ func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error { return errAlreadyDialing case peers[n.ID()] != nil: return errAlreadyConnected - case s.ntab != nil && n.ID() == s.ntab.Self().ID(): + case n.ID() == s.self: return errSelf case s.netrestrict != nil && !s.netrestrict.Contains(n.IP()): return errNotWhitelisted @@ -349,7 +350,7 @@ func (t *dialTask) dial(srv *Server, dest *enode.Node) error { if err != nil { return &dialError{err} } - mfd := newMeteredConn(fd, false) + mfd := newMeteredConn(fd, false, dest.IP()) return srv.SetupConn(mfd, t.flags, dest) } diff --git a/p2p/dial_test.go b/p2p/dial_test.go index 2de2c59995..f41ab77524 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -89,7 +89,7 @@ func (t fakeTable) ReadRandomNodes(buf []*enode.Node) int { return copy(buf, t) // This test checks that dynamic dials are launched from discovery results. func TestDialStateDynDial(t *testing.T) { runDialTest(t, dialtest{ - init: newDialState(nil, nil, fakeTable{}, 5, nil), + init: newDialState(enode.ID{}, nil, nil, fakeTable{}, 5, nil), rounds: []round{ // A discovery query is launched. { @@ -236,7 +236,7 @@ func TestDialStateDynDialBootnode(t *testing.T) { newNode(uintID(8), nil), } runDialTest(t, dialtest{ - init: newDialState(nil, bootnodes, table, 5, nil), + init: newDialState(enode.ID{}, nil, bootnodes, table, 5, nil), rounds: []round{ // 2 dynamic dials attempted, bootnodes pending fallback interval { @@ -324,7 +324,7 @@ func TestDialStateDynDialFromTable(t *testing.T) { } runDialTest(t, dialtest{ - init: newDialState(nil, nil, table, 10, nil), + init: newDialState(enode.ID{}, nil, nil, table, 10, nil), rounds: []round{ // 5 out of 8 of the nodes returned by ReadRandomNodes are dialed. { @@ -430,7 +430,7 @@ func TestDialStateNetRestrict(t *testing.T) { restrict.Add("127.0.2.0/24") runDialTest(t, dialtest{ - init: newDialState(nil, nil, table, 10, restrict), + init: newDialState(enode.ID{}, nil, nil, table, 10, restrict), rounds: []round{ { new: []task{ @@ -453,7 +453,7 @@ func TestDialStateStaticDial(t *testing.T) { } runDialTest(t, dialtest{ - init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil), rounds: []round{ // Static dials are launched for the nodes that // aren't yet connected. @@ -557,7 +557,7 @@ func TestDialStaticAfterReset(t *testing.T) { }, } dTest := dialtest{ - init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil), rounds: rounds, } runDialTest(t, dTest) @@ -578,7 +578,7 @@ func TestDialStateCache(t *testing.T) { } runDialTest(t, dialtest{ - init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil), rounds: []round{ // Static dials are launched for the nodes that // aren't yet connected. @@ -640,7 +640,7 @@ func TestDialStateCache(t *testing.T) { func TestDialResolve(t *testing.T) { resolved := newNode(uintID(1), net.IP{127, 0, 55, 234}) table := &resolveMock{answer: resolved} - state := newDialState(nil, nil, table, 0, nil) + state := newDialState(enode.ID{}, nil, nil, table, 0, nil) // Check that the task is generated with an incomplete ID. dest := newNode(uintID(1), nil) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 7a3e41de18..afd4c9a272 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -72,21 +72,20 @@ type Table struct { ips netutil.DistinctNetSet db *enode.DB // database of known nodes + net transport refreshReq chan chan struct{} initDone chan struct{} closeReq chan struct{} closed chan struct{} nodeAddedHook func(*node) // for testing - - net transport - self *node // metadata of the local node } // transport is implemented by the UDP transport. // it is an interface so we can test without opening lots of UDP // sockets and without generating a private key. type transport interface { + self() *enode.Node ping(enode.ID, *net.UDPAddr) error findnode(toid enode.ID, addr *net.UDPAddr, target encPubkey) ([]*node, error) close() @@ -100,11 +99,10 @@ type bucket struct { ips netutil.DistinctNetSet } -func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.Node) (*Table, error) { +func newTable(t transport, db *enode.DB, bootnodes []*enode.Node) (*Table, error) { tab := &Table{ net: t, db: db, - self: wrapNode(self), refreshReq: make(chan chan struct{}), initDone: make(chan struct{}), closeReq: make(chan struct{}), @@ -127,6 +125,10 @@ func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.No return tab, nil } +func (tab *Table) self() *enode.Node { + return tab.net.self() +} + func (tab *Table) seedRand() { var b [8]byte crand.Read(b[:]) @@ -136,11 +138,6 @@ func (tab *Table) seedRand() { tab.mutex.Unlock() } -// Self returns the local node. -func (tab *Table) Self() *enode.Node { - return unwrapNode(tab.self) -} - // ReadRandomNodes fills the given slice with random nodes from the table. The results // are guaranteed to be unique for a single invocation, no node will appear twice. func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) { @@ -183,6 +180,10 @@ func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) { // Close terminates the network listener and flushes the node database. func (tab *Table) Close() { + if tab.net != nil { + tab.net.close() + } + select { case <-tab.closed: // already closed. @@ -257,7 +258,7 @@ func (tab *Table) lookup(targetKey encPubkey, refreshIfEmpty bool) []*node { ) // don't query further if we hit ourself. // unlikely to happen often in practice. - asked[tab.self.ID()] = true + asked[tab.self().ID()] = true for { tab.mutex.Lock() @@ -340,8 +341,8 @@ func (tab *Table) loop() { revalidate = time.NewTimer(tab.nextRevalidateTime()) refresh = time.NewTicker(refreshInterval) copyNodes = time.NewTicker(copyNodesInterval) - revalidateDone = make(chan struct{}) refreshDone = make(chan struct{}) // where doRefresh reports completion + revalidateDone chan struct{} // where doRevalidate reports completion waiting = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs ) defer refresh.Stop() @@ -372,9 +373,11 @@ loop: } waiting, refreshDone = nil, nil case <-revalidate.C: + revalidateDone = make(chan struct{}) go tab.doRevalidate(revalidateDone) case <-revalidateDone: revalidate.Reset(tab.nextRevalidateTime()) + revalidateDone = nil case <-copyNodes.C: go tab.copyLiveNodes() case <-tab.closeReq: @@ -382,15 +385,15 @@ loop: } } - if tab.net != nil { - tab.net.close() - } if refreshDone != nil { <-refreshDone } for _, ch := range waiting { close(ch) } + if revalidateDone != nil { + <-revalidateDone + } close(tab.closed) } @@ -408,7 +411,7 @@ func (tab *Table) doRefresh(done chan struct{}) { // Run self lookup to discover new neighbor nodes. // We can only do this if we have a secp256k1 identity. var key ecdsa.PublicKey - if err := tab.self.Load((*enode.Secp256k1)(&key)); err == nil { + if err := tab.self().Load((*enode.Secp256k1)(&key)); err == nil { tab.lookup(encodePubkey(&key), false) } @@ -530,7 +533,7 @@ func (tab *Table) len() (n int) { // bucket returns the bucket for the given node ID hash. func (tab *Table) bucket(id enode.ID) *bucket { - d := enode.LogDist(tab.self.ID(), id) + d := enode.LogDist(tab.self().ID(), id) if d <= bucketMinDistance { return tab.buckets[0] } @@ -543,7 +546,7 @@ func (tab *Table) bucket(id enode.ID) *bucket { // // The caller must not hold tab.mutex. func (tab *Table) add(n *node) { - if n.ID() == tab.self.ID() { + if n.ID() == tab.self().ID() { return } @@ -576,7 +579,7 @@ func (tab *Table) stuff(nodes []*node) { defer tab.mutex.Unlock() for _, n := range nodes { - if n.ID() == tab.self.ID() { + if n.ID() == tab.self().ID() { continue // don't add self } b := tab.bucket(n.ID()) diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index e8631024b6..6b4cd2d186 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -141,7 +141,7 @@ func TestTable_IPLimit(t *testing.T) { defer db.Close() for i := 0; i < tableIPLimit+1; i++ { - n := nodeAtDistance(tab.self.ID(), i, net.IP{172, 0, 1, byte(i)}) + n := nodeAtDistance(tab.self().ID(), i, net.IP{172, 0, 1, byte(i)}) tab.add(n) } if tab.len() > tableIPLimit { @@ -158,7 +158,7 @@ func TestTable_BucketIPLimit(t *testing.T) { d := 3 for i := 0; i < bucketIPLimit+1; i++ { - n := nodeAtDistance(tab.self.ID(), d, net.IP{172, 0, 1, byte(i)}) + n := nodeAtDistance(tab.self().ID(), d, net.IP{172, 0, 1, byte(i)}) tab.add(n) } if tab.len() > bucketIPLimit { @@ -240,7 +240,7 @@ func TestTable_ReadRandomNodesGetAll(t *testing.T) { for i := 0; i < len(buf); i++ { ld := cfg.Rand.Intn(len(tab.buckets)) - tab.stuff([]*node{nodeAtDistance(tab.self.ID(), ld, intIP(ld))}) + tab.stuff([]*node{nodeAtDistance(tab.self().ID(), ld, intIP(ld))}) } gotN := tab.ReadRandomNodes(buf) if gotN != tab.len() { @@ -510,6 +510,10 @@ type preminedTestnet struct { dists [hashBits + 1][]encPubkey } +func (tn *preminedTestnet) self() *enode.Node { + return nullNode +} + func (tn *preminedTestnet) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) { // current log distance is encoded in port number // fmt.Println("findnode query at dist", toaddr.Port) diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 05ae0b6c07..d415194521 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -28,12 +28,17 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" ) -func newTestTable(t transport) (*Table, *enode.DB) { +var nullNode *enode.Node + +func init() { var r enr.Record r.Set(enr.IP{0, 0, 0, 0}) - n := enode.SignNull(&r, enode.ID{}) + nullNode = enode.SignNull(&r, enode.ID{}) +} + +func newTestTable(t transport) (*Table, *enode.DB) { db, _ := enode.OpenDB("") - tab, _ := newTable(t, n, db, nil) + tab, _ := newTable(t, db, nil) return tab, db } @@ -70,10 +75,10 @@ func intIP(i int) net.IP { // fillBucket inserts nodes into the given bucket until it is full. func fillBucket(tab *Table, n *node) (last *node) { - ld := enode.LogDist(tab.self.ID(), n.ID()) + ld := enode.LogDist(tab.self().ID(), n.ID()) b := tab.bucket(n.ID()) for len(b.entries) < bucketSize { - b.entries = append(b.entries, nodeAtDistance(tab.self.ID(), ld, intIP(ld))) + b.entries = append(b.entries, nodeAtDistance(tab.self().ID(), ld, intIP(ld))) } return b.entries[bucketSize-1] } @@ -81,15 +86,25 @@ func fillBucket(tab *Table, n *node) (last *node) { type pingRecorder struct { mu sync.Mutex dead, pinged map[enode.ID]bool + n *enode.Node } func newPingRecorder() *pingRecorder { + var r enr.Record + r.Set(enr.IP{0, 0, 0, 0}) + n := enode.SignNull(&r, enode.ID{}) + return &pingRecorder{ dead: make(map[enode.ID]bool), pinged: make(map[enode.ID]bool), + n: n, } } +func (t *pingRecorder) self() *enode.Node { + return nullNode +} + func (t *pingRecorder) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) { return nil, nil } diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index 45fcce282c..37a0449021 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -23,12 +23,12 @@ import ( "errors" "fmt" "net" + "sync" "time" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/rlp" ) @@ -118,9 +118,11 @@ type ( ) func makeEndpoint(addr *net.UDPAddr, tcpPort uint16) rpcEndpoint { - ip := addr.IP.To4() - if ip == nil { - ip = addr.IP.To16() + ip := net.IP{} + if ip4 := addr.IP.To4(); ip4 != nil { + ip = ip4 + } else if ip6 := addr.IP.To16(); ip6 != nil { + ip = ip6 } return rpcEndpoint{IP: ip, UDP: uint16(addr.Port), TCP: tcpPort} } @@ -165,20 +167,19 @@ type conn interface { LocalAddr() net.Addr } -// udp implements the RPC protocol. +// udp implements the discovery v4 UDP wire protocol. type udp struct { conn conn netrestrict *netutil.Netlist priv *ecdsa.PrivateKey - ourEndpoint rpcEndpoint + localNode *enode.LocalNode + db *enode.DB + tab *Table + wg sync.WaitGroup addpending chan *pending gotreply chan reply - - closing chan struct{} - nat nat.Interface - - *Table + closing chan struct{} } // pending represents a pending reply. @@ -230,60 +231,57 @@ type Config struct { PrivateKey *ecdsa.PrivateKey // These settings are optional: - AnnounceAddr *net.UDPAddr // local address announced in the DHT - NodeDBPath string // if set, the node database is stored at this filesystem location - NetRestrict *netutil.Netlist // network whitelist - Bootnodes []*enode.Node // list of bootstrap nodes - Unhandled chan<- ReadPacket // unhandled packets are sent on this channel + NetRestrict *netutil.Netlist // network whitelist + Bootnodes []*enode.Node // list of bootstrap nodes + Unhandled chan<- ReadPacket // unhandled packets are sent on this channel } // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(c conn, cfg Config) (*Table, error) { - tab, _, err := newUDP(c, cfg) +func ListenUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, error) { + tab, _, err := newUDP(c, ln, cfg) if err != nil { return nil, err } - log.Info("UDP listener up", "self", tab.self) return tab, nil } -func newUDP(c conn, cfg Config) (*Table, *udp, error) { - realaddr := c.LocalAddr().(*net.UDPAddr) - if cfg.AnnounceAddr != nil { - realaddr = cfg.AnnounceAddr - } - self := enode.NewV4(&cfg.PrivateKey.PublicKey, realaddr.IP, realaddr.Port, realaddr.Port) - db, err := enode.OpenDB(cfg.NodeDBPath) - if err != nil { - return nil, nil, err - } - +func newUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, *udp, error) { udp := &udp{ conn: c, priv: cfg.PrivateKey, netrestrict: cfg.NetRestrict, + localNode: ln, + db: ln.Database(), closing: make(chan struct{}), gotreply: make(chan reply), addpending: make(chan *pending), } - // TODO: separate TCP port - udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port)) - tab, err := newTable(udp, self, db, cfg.Bootnodes) + tab, err := newTable(udp, ln.Database(), cfg.Bootnodes) if err != nil { return nil, nil, err } - udp.Table = tab + udp.tab = tab + udp.wg.Add(2) go udp.loop() go udp.readLoop(cfg.Unhandled) - return udp.Table, udp, nil + return udp.tab, udp, nil +} + +func (t *udp) self() *enode.Node { + return t.localNode.Node() } func (t *udp) close() { close(t.closing) t.conn.Close() - t.db.Close() - // TODO: wait for the loops to end. + t.wg.Wait() +} + +func (t *udp) ourEndpoint() rpcEndpoint { + n := t.self() + a := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} + return makeEndpoint(a, uint16(n.TCP())) } // ping sends a ping message to the given node and waits for a reply. @@ -296,7 +294,7 @@ func (t *udp) ping(toid enode.ID, toaddr *net.UDPAddr) error { func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-chan error { req := &ping{ Version: 4, - From: t.ourEndpoint, + From: t.ourEndpoint(), To: makeEndpoint(toaddr, 0), // TODO: maybe use known TCP port from DB Expiration: uint64(time.Now().Add(expiration).Unix()), } @@ -313,6 +311,7 @@ func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-ch } return ok }) + t.localNode.UDPContact(toaddr) t.write(toaddr, req.name(), packet) return errc } @@ -381,6 +380,8 @@ func (t *udp) handleReply(from enode.ID, ptype byte, req packet) bool { // loop runs in its own goroutine. it keeps track of // the refresh timer and the pending reply queue. func (t *udp) loop() { + defer t.wg.Done() + var ( plist = list.New() timeout = time.NewTimer(0) @@ -542,10 +543,11 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (packet, // readLoop runs in its own goroutine. it handles incoming UDP packets. func (t *udp) readLoop(unhandled chan<- ReadPacket) { - defer t.conn.Close() + defer t.wg.Done() if unhandled != nil { defer close(unhandled) } + // Discovery packets are defined to be no larger than 1280 bytes. // Packets larger than this size will be cut at the end and treated // as invalid because their hash won't match. @@ -629,10 +631,11 @@ func (req *ping) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte n := wrapNode(enode.NewV4(key, from.IP, int(req.From.TCP), from.Port)) t.handleReply(n.ID(), pingPacket, req) if time.Since(t.db.LastPongReceived(n.ID())) > bondExpiration { - t.sendPing(n.ID(), from, func() { t.addThroughPing(n) }) + t.sendPing(n.ID(), from, func() { t.tab.addThroughPing(n) }) } else { - t.addThroughPing(n) + t.tab.addThroughPing(n) } + t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) t.db.UpdateLastPingReceived(n.ID(), time.Now()) return nil } @@ -647,6 +650,7 @@ func (req *pong) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte if !t.handleReply(fromID, pongPacket, req) { return errUnsolicitedReply } + t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) t.db.UpdateLastPongReceived(fromID, time.Now()) return nil } @@ -668,9 +672,9 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac [] return errUnknownNode } target := enode.ID(crypto.Keccak256Hash(req.Target[:])) - t.mutex.Lock() - closest := t.closest(target, bucketSize).entries - t.mutex.Unlock() + t.tab.mutex.Lock() + closest := t.tab.closest(target, bucketSize).entries + t.tab.mutex.Unlock() p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())} var sent bool diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go index da95c4f5c4..a4ddaf750d 100644 --- a/p2p/discover/udp_test.go +++ b/p2p/discover/udp_test.go @@ -71,7 +71,9 @@ func newUDPTest(t *testing.T) *udpTest { remotekey: newkey(), remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, } - test.table, test.udp, _ = newUDP(test.pipe, Config{PrivateKey: test.localkey}) + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, test.localkey) + test.table, test.udp, _ = newUDP(test.pipe, ln, Config{PrivateKey: test.localkey}) // Wait for initial refresh so the table doesn't send unexpected findnode. <-test.table.initDone return test @@ -355,12 +357,13 @@ func TestUDP_successfulPing(t *testing.T) { // remote is unknown, the table pings back. hash, _ := test.waitPacketOut(func(p *ping) error { - if !reflect.DeepEqual(p.From, test.udp.ourEndpoint) { - t.Errorf("got ping.From %v, want %v", p.From, test.udp.ourEndpoint) + if !reflect.DeepEqual(p.From, test.udp.ourEndpoint()) { + t.Errorf("got ping.From %#v, want %#v", p.From, test.udp.ourEndpoint()) } wantTo := rpcEndpoint{ // The mirrored UDP address is the UDP packet sender. - IP: test.remoteaddr.IP, UDP: uint16(test.remoteaddr.Port), + IP: test.remoteaddr.IP, + UDP: uint16(test.remoteaddr.Port), TCP: 0, } if !reflect.DeepEqual(p.To, wantTo) { diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index 49e1cb811a..ff5ed983ba 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -230,7 +230,8 @@ type udp struct { } // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { +func ListenUDP(priv *ecdsa.PrivateKey, conn conn, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { + realaddr := conn.LocalAddr().(*net.UDPAddr) transport, err := listenUDP(priv, conn, realaddr) if err != nil { return nil, err diff --git a/p2p/enode/localnode.go b/p2p/enode/localnode.go new file mode 100644 index 0000000000..623f8eae18 --- /dev/null +++ b/p2p/enode/localnode.go @@ -0,0 +1,246 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enode + +import ( + "crypto/ecdsa" + "fmt" + "net" + "reflect" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/netutil" +) + +const ( + // IP tracker configuration + iptrackMinStatements = 10 + iptrackWindow = 5 * time.Minute + iptrackContactWindow = 10 * time.Minute +) + +// LocalNode produces the signed node record of a local node, i.e. a node run in the +// current process. Setting ENR entries via the Set method updates the record. A new version +// of the record is signed on demand when the Node method is called. +type LocalNode struct { + cur atomic.Value // holds a non-nil node pointer while the record is up-to-date. + id ID + key *ecdsa.PrivateKey + db *DB + + // everything below is protected by a lock + mu sync.Mutex + seq uint64 + entries map[string]enr.Entry + udpTrack *netutil.IPTracker // predicts external UDP endpoint + staticIP net.IP + fallbackIP net.IP + fallbackUDP int +} + +// NewLocalNode creates a local node. +func NewLocalNode(db *DB, key *ecdsa.PrivateKey) *LocalNode { + ln := &LocalNode{ + id: PubkeyToIDV4(&key.PublicKey), + db: db, + key: key, + udpTrack: netutil.NewIPTracker(iptrackWindow, iptrackContactWindow, iptrackMinStatements), + entries: make(map[string]enr.Entry), + } + ln.seq = db.localSeq(ln.id) + ln.invalidate() + return ln +} + +// Database returns the node database associated with the local node. +func (ln *LocalNode) Database() *DB { + return ln.db +} + +// Node returns the current version of the local node record. +func (ln *LocalNode) Node() *Node { + n := ln.cur.Load().(*Node) + if n != nil { + return n + } + // Record was invalidated, sign a new copy. + ln.mu.Lock() + defer ln.mu.Unlock() + ln.sign() + return ln.cur.Load().(*Node) +} + +// ID returns the local node ID. +func (ln *LocalNode) ID() ID { + return ln.id +} + +// Set puts the given entry into the local record, overwriting +// any existing value. +func (ln *LocalNode) Set(e enr.Entry) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.set(e) +} + +func (ln *LocalNode) set(e enr.Entry) { + val, exists := ln.entries[e.ENRKey()] + if !exists || !reflect.DeepEqual(val, e) { + ln.entries[e.ENRKey()] = e + ln.invalidate() + } +} + +// Delete removes the given entry from the local record. +func (ln *LocalNode) Delete(e enr.Entry) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.delete(e) +} + +func (ln *LocalNode) delete(e enr.Entry) { + _, exists := ln.entries[e.ENRKey()] + if exists { + delete(ln.entries, e.ENRKey()) + ln.invalidate() + } +} + +// SetStaticIP sets the local IP to the given one unconditionally. +// This disables endpoint prediction. +func (ln *LocalNode) SetStaticIP(ip net.IP) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.staticIP = ip + ln.updateEndpoints() +} + +// SetFallbackIP sets the last-resort IP address. This address is used +// if no endpoint prediction can be made and no static IP is set. +func (ln *LocalNode) SetFallbackIP(ip net.IP) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.fallbackIP = ip + ln.updateEndpoints() +} + +// SetFallbackUDP sets the last-resort UDP port. This port is used +// if no endpoint prediction can be made. +func (ln *LocalNode) SetFallbackUDP(port int) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.fallbackUDP = port + ln.updateEndpoints() +} + +// UDPEndpointStatement should be called whenever a statement about the local node's +// UDP endpoint is received. It feeds the local endpoint predictor. +func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint *net.UDPAddr) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.udpTrack.AddStatement(fromaddr.String(), endpoint.String()) + ln.updateEndpoints() +} + +// UDPContact should be called whenever the local node has announced itself to another node +// via UDP. It feeds the local endpoint predictor. +func (ln *LocalNode) UDPContact(toaddr *net.UDPAddr) { + ln.mu.Lock() + defer ln.mu.Unlock() + + ln.udpTrack.AddContact(toaddr.String()) + ln.updateEndpoints() +} + +func (ln *LocalNode) updateEndpoints() { + // Determine the endpoints. + newIP := ln.fallbackIP + newUDP := ln.fallbackUDP + if ln.staticIP != nil { + newIP = ln.staticIP + } else if ip, port := predictAddr(ln.udpTrack); ip != nil { + newIP = ip + newUDP = port + } + + // Update the record. + if newIP != nil && !newIP.IsUnspecified() { + ln.set(enr.IP(newIP)) + if newUDP != 0 { + ln.set(enr.UDP(newUDP)) + } else { + ln.delete(enr.UDP(0)) + } + } else { + ln.delete(enr.IP{}) + } +} + +// predictAddr wraps IPTracker.PredictEndpoint, converting from its string-based +// endpoint representation to IP and port types. +func predictAddr(t *netutil.IPTracker) (net.IP, int) { + ep := t.PredictEndpoint() + if ep == "" { + return nil, 0 + } + ipString, portString, _ := net.SplitHostPort(ep) + ip := net.ParseIP(ipString) + port, _ := strconv.Atoi(portString) + return ip, port +} + +func (ln *LocalNode) invalidate() { + ln.cur.Store((*Node)(nil)) +} + +func (ln *LocalNode) sign() { + if n := ln.cur.Load().(*Node); n != nil { + return // no changes + } + + var r enr.Record + for _, e := range ln.entries { + r.Set(e) + } + ln.bumpSeq() + r.SetSeq(ln.seq) + if err := SignV4(&r, ln.key); err != nil { + panic(fmt.Errorf("enode: can't sign record: %v", err)) + } + n, err := New(ValidSchemes, &r) + if err != nil { + panic(fmt.Errorf("enode: can't verify local record: %v", err)) + } + ln.cur.Store(n) + log.Info("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IP(), "udp", n.UDP(), "tcp", n.TCP()) +} + +func (ln *LocalNode) bumpSeq() { + ln.seq++ + ln.db.storeLocalSeq(ln.id, ln.seq) +} diff --git a/p2p/enode/localnode_test.go b/p2p/enode/localnode_test.go new file mode 100644 index 0000000000..f5e3496d63 --- /dev/null +++ b/p2p/enode/localnode_test.go @@ -0,0 +1,76 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package enode + +import ( + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/enr" +) + +func newLocalNodeForTesting() (*LocalNode, *DB) { + db, _ := OpenDB("") + key, _ := crypto.GenerateKey() + return NewLocalNode(db, key), db +} + +func TestLocalNode(t *testing.T) { + ln, db := newLocalNodeForTesting() + defer db.Close() + + if ln.Node().ID() != ln.ID() { + t.Fatal("inconsistent ID") + } + + ln.Set(enr.WithEntry("x", uint(3))) + var x uint + if err := ln.Node().Load(enr.WithEntry("x", &x)); err != nil { + t.Fatal("can't load entry 'x':", err) + } else if x != 3 { + t.Fatal("wrong value for entry 'x':", x) + } +} + +func TestLocalNodeSeqPersist(t *testing.T) { + ln, db := newLocalNodeForTesting() + defer db.Close() + + if s := ln.Node().Seq(); s != 1 { + t.Fatalf("wrong initial seq %d, want 1", s) + } + ln.Set(enr.WithEntry("x", uint(1))) + if s := ln.Node().Seq(); s != 2 { + t.Fatalf("wrong seq %d after set, want 2", s) + } + + // Create a new instance, it should reload the sequence number. + // The number increases just after that because a new record is + // created without the "x" entry. + ln2 := NewLocalNode(db, ln.key) + if s := ln2.Node().Seq(); s != 3 { + t.Fatalf("wrong seq %d on new instance, want 3", s) + } + + // Create a new instance with a different node key on the same database. + // This should reset the sequence number. + key, _ := crypto.GenerateKey() + ln3 := NewLocalNode(db, key) + if s := ln3.Node().Seq(); s != 1 { + t.Fatalf("wrong seq %d on instance with changed key, want 1", s) + } +} diff --git a/p2p/enode/node.go b/p2p/enode/node.go index 84088fcd26..b454ab2554 100644 --- a/p2p/enode/node.go +++ b/p2p/enode/node.go @@ -98,6 +98,13 @@ func (n *Node) Pubkey() *ecdsa.PublicKey { return &key } +// Record returns the node's record. The return value is a copy and may +// be modified by the caller. +func (n *Node) Record() *enr.Record { + cpy := n.r + return &cpy +} + // checks whether n is a valid complete node. func (n *Node) ValidateComplete() error { if n.Incomplete() { diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index a929b75d7b..7ee0c09a93 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -35,11 +35,24 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) +// Keys in the node database. +const ( + dbVersionKey = "version" // Version of the database to flush if changes + dbItemPrefix = "n:" // Identifier to prefix node entries with + + dbDiscoverRoot = ":discover" + dbDiscoverSeq = dbDiscoverRoot + ":seq" + dbDiscoverPing = dbDiscoverRoot + ":lastping" + dbDiscoverPong = dbDiscoverRoot + ":lastpong" + dbDiscoverFindFails = dbDiscoverRoot + ":findfail" + dbLocalRoot = ":local" + dbLocalSeq = dbLocalRoot + ":seq" +) + var ( - nodeDBNilID = ID{} // Special node ID to use as a nil element. - nodeDBNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped. - nodeDBCleanupCycle = time.Hour // Time period for running the expiration task. - nodeDBVersion = 6 + dbNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped. + dbCleanupCycle = time.Hour // Time period for running the expiration task. + dbVersion = 7 ) // DB is the node database, storing previously seen nodes and any collected metadata about @@ -50,17 +63,6 @@ type DB struct { quit chan struct{} // Channel to signal the expiring thread to stop } -// Schema layout for the node database -var ( - nodeDBVersionKey = []byte("version") // Version of the database to flush if changes - nodeDBItemPrefix = []byte("n:") // Identifier to prefix node entries with - - nodeDBDiscoverRoot = ":discover" - nodeDBDiscoverPing = nodeDBDiscoverRoot + ":lastping" - nodeDBDiscoverPong = nodeDBDiscoverRoot + ":lastpong" - nodeDBDiscoverFindFails = nodeDBDiscoverRoot + ":findfail" -) - // OpenDB opens a node database for storing and retrieving infos about known peers in the // network. If no path is given an in-memory, temporary database is constructed. func OpenDB(path string) (*DB, error) { @@ -93,13 +95,13 @@ func newPersistentDB(path string) (*DB, error) { // The nodes contained in the cache correspond to a certain protocol version. // Flush all nodes if the version doesn't match. currentVer := make([]byte, binary.MaxVarintLen64) - currentVer = currentVer[:binary.PutVarint(currentVer, int64(nodeDBVersion))] + currentVer = currentVer[:binary.PutVarint(currentVer, int64(dbVersion))] - blob, err := db.Get(nodeDBVersionKey, nil) + blob, err := db.Get([]byte(dbVersionKey), nil) switch err { case leveldb.ErrNotFound: // Version not found (i.e. empty cache), insert it - if err := db.Put(nodeDBVersionKey, currentVer, nil); err != nil { + if err := db.Put([]byte(dbVersionKey), currentVer, nil); err != nil { db.Close() return nil, err } @@ -120,28 +122,27 @@ func newPersistentDB(path string) (*DB, error) { // makeKey generates the leveldb key-blob from a node id and its particular // field of interest. func makeKey(id ID, field string) []byte { - if bytes.Equal(id[:], nodeDBNilID[:]) { + if (id == ID{}) { return []byte(field) } - return append(nodeDBItemPrefix, append(id[:], field...)...) + return append([]byte(dbItemPrefix), append(id[:], field...)...) } // splitKey tries to split a database key into a node id and a field part. func splitKey(key []byte) (id ID, field string) { // If the key is not of a node, return it plainly - if !bytes.HasPrefix(key, nodeDBItemPrefix) { + if !bytes.HasPrefix(key, []byte(dbItemPrefix)) { return ID{}, string(key) } // Otherwise split the id and field - item := key[len(nodeDBItemPrefix):] + item := key[len(dbItemPrefix):] copy(id[:], item[:len(id)]) field = string(item[len(id):]) return id, field } -// fetchInt64 retrieves an integer instance associated with a particular -// database key. +// fetchInt64 retrieves an integer associated with a particular key. func (db *DB) fetchInt64(key []byte) int64 { blob, err := db.lvl.Get(key, nil) if err != nil { @@ -154,18 +155,33 @@ func (db *DB) fetchInt64(key []byte) int64 { return val } -// storeInt64 update a specific database entry to the current time instance as a -// unix timestamp. +// storeInt64 stores an integer in the given key. func (db *DB) storeInt64(key []byte, n int64) error { blob := make([]byte, binary.MaxVarintLen64) blob = blob[:binary.PutVarint(blob, n)] + return db.lvl.Put(key, blob, nil) +} +// fetchUint64 retrieves an integer associated with a particular key. +func (db *DB) fetchUint64(key []byte) uint64 { + blob, err := db.lvl.Get(key, nil) + if err != nil { + return 0 + } + val, _ := binary.Uvarint(blob) + return val +} + +// storeUint64 stores an integer in the given key. +func (db *DB) storeUint64(key []byte, n uint64) error { + blob := make([]byte, binary.MaxVarintLen64) + blob = blob[:binary.PutUvarint(blob, n)] return db.lvl.Put(key, blob, nil) } // Node retrieves a node with a given id from the database. func (db *DB) Node(id ID) *Node { - blob, err := db.lvl.Get(makeKey(id, nodeDBDiscoverRoot), nil) + blob, err := db.lvl.Get(makeKey(id, dbDiscoverRoot), nil) if err != nil { return nil } @@ -184,11 +200,31 @@ func mustDecodeNode(id, data []byte) *Node { // UpdateNode inserts - potentially overwriting - a node into the peer database. func (db *DB) UpdateNode(node *Node) error { + if node.Seq() < db.NodeSeq(node.ID()) { + return nil + } blob, err := rlp.EncodeToBytes(&node.r) if err != nil { return err } - return db.lvl.Put(makeKey(node.ID(), nodeDBDiscoverRoot), blob, nil) + if err := db.lvl.Put(makeKey(node.ID(), dbDiscoverRoot), blob, nil); err != nil { + return err + } + return db.storeUint64(makeKey(node.ID(), dbDiscoverSeq), node.Seq()) +} + +// NodeSeq returns the stored record sequence number of the given node. +func (db *DB) NodeSeq(id ID) uint64 { + return db.fetchUint64(makeKey(id, dbDiscoverSeq)) +} + +// Resolve returns the stored record of the node if it has a larger sequence +// number than n. +func (db *DB) Resolve(n *Node) *Node { + if n.Seq() > db.NodeSeq(n.ID()) { + return n + } + return db.Node(n.ID()) } // DeleteNode deletes all information/keys associated with a node. @@ -218,7 +254,7 @@ func (db *DB) ensureExpirer() { // expirer should be started in a go routine, and is responsible for looping ad // infinitum and dropping stale data from the database. func (db *DB) expirer() { - tick := time.NewTicker(nodeDBCleanupCycle) + tick := time.NewTicker(dbCleanupCycle) defer tick.Stop() for { select { @@ -235,7 +271,7 @@ func (db *DB) expirer() { // expireNodes iterates over the database and deletes all nodes that have not // been seen (i.e. received a pong from) for some allotted time. func (db *DB) expireNodes() error { - threshold := time.Now().Add(-nodeDBNodeExpiration) + threshold := time.Now().Add(-dbNodeExpiration) // Find discovered nodes that are older than the allowance it := db.lvl.NewIterator(nil, nil) @@ -244,7 +280,7 @@ func (db *DB) expireNodes() error { for it.Next() { // Skip the item if not a discovery node id, field := splitKey(it.Key()) - if field != nodeDBDiscoverRoot { + if field != dbDiscoverRoot { continue } // Skip the node if not expired yet (and not self) @@ -260,34 +296,44 @@ func (db *DB) expireNodes() error { // LastPingReceived retrieves the time of the last ping packet received from // a remote node. func (db *DB) LastPingReceived(id ID) time.Time { - return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPing)), 0) + return time.Unix(db.fetchInt64(makeKey(id, dbDiscoverPing)), 0) } // UpdateLastPingReceived updates the last time we tried contacting a remote node. func (db *DB) UpdateLastPingReceived(id ID, instance time.Time) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix()) + return db.storeInt64(makeKey(id, dbDiscoverPing), instance.Unix()) } // LastPongReceived retrieves the time of the last successful pong from remote node. func (db *DB) LastPongReceived(id ID) time.Time { // Launch expirer db.ensureExpirer() - return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0) + return time.Unix(db.fetchInt64(makeKey(id, dbDiscoverPong)), 0) } // UpdateLastPongReceived updates the last pong time of a node. func (db *DB) UpdateLastPongReceived(id ID, instance time.Time) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix()) + return db.storeInt64(makeKey(id, dbDiscoverPong), instance.Unix()) } // FindFails retrieves the number of findnode failures since bonding. func (db *DB) FindFails(id ID) int { - return int(db.fetchInt64(makeKey(id, nodeDBDiscoverFindFails))) + return int(db.fetchInt64(makeKey(id, dbDiscoverFindFails))) } // UpdateFindFails updates the number of findnode failures since bonding. func (db *DB) UpdateFindFails(id ID, fails int) error { - return db.storeInt64(makeKey(id, nodeDBDiscoverFindFails), int64(fails)) + return db.storeInt64(makeKey(id, dbDiscoverFindFails), int64(fails)) +} + +// LocalSeq retrieves the local record sequence counter. +func (db *DB) localSeq(id ID) uint64 { + return db.fetchUint64(makeKey(id, dbLocalSeq)) +} + +// storeLocalSeq stores the local record sequence counter. +func (db *DB) storeLocalSeq(id ID, n uint64) { + db.storeUint64(makeKey(id, dbLocalSeq), n) } // QuerySeeds retrieves random nodes to be used as potential seed nodes @@ -309,7 +355,7 @@ seek: ctr := id[0] rand.Read(id[:]) id[0] = ctr + id[0]%16 - it.Seek(makeKey(id, nodeDBDiscoverRoot)) + it.Seek(makeKey(id, dbDiscoverRoot)) n := nextNode(it) if n == nil { @@ -334,7 +380,7 @@ seek: func nextNode(it iterator.Iterator) *Node { for end := false; !end; end = !it.Next() { id, field := splitKey(it.Key()) - if field != nodeDBDiscoverRoot { + if field != dbDiscoverRoot { continue } return mustDecodeNode(id[:], it.Value()) diff --git a/p2p/enode/nodedb_test.go b/p2p/enode/nodedb_test.go index b476a3439d..96794827cf 100644 --- a/p2p/enode/nodedb_test.go +++ b/p2p/enode/nodedb_test.go @@ -332,7 +332,7 @@ var nodeDBExpirationNodes = []struct { 30303, 30303, ), - pong: time.Now().Add(-nodeDBNodeExpiration + time.Minute), + pong: time.Now().Add(-dbNodeExpiration + time.Minute), exp: false, }, { node: NewV4( @@ -341,7 +341,7 @@ var nodeDBExpirationNodes = []struct { 30303, 30303, ), - pong: time.Now().Add(-nodeDBNodeExpiration - time.Minute), + pong: time.Now().Add(-dbNodeExpiration - time.Minute), exp: true, }, } diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index 251caf4585..444820c158 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -156,7 +156,7 @@ func (r *Record) Set(e Entry) { } func (r *Record) invalidate() { - if r.signature == nil { + if r.signature != nil { r.seq++ } r.signature = nil diff --git a/p2p/enr/enr_test.go b/p2p/enr/enr_test.go index 9bf22478dd..449c898a84 100644 --- a/p2p/enr/enr_test.go +++ b/p2p/enr/enr_test.go @@ -169,6 +169,18 @@ func TestDirty(t *testing.T) { } } +func TestSeq(t *testing.T) { + var r Record + + assert.Equal(t, uint64(0), r.Seq()) + r.Set(UDP(1)) + assert.Equal(t, uint64(0), r.Seq()) + signTest([]byte{5}, &r) + assert.Equal(t, uint64(0), r.Seq()) + r.Set(UDP(2)) + assert.Equal(t, uint64(1), r.Seq()) +} + // TestGetSetOverwrite tests value overwrite when setting a new value with an existing key in record. func TestGetSetOverwrite(t *testing.T) { var r Record diff --git a/p2p/metrics.go b/p2p/metrics.go index 2d52fd1fd1..8df82bb074 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -19,53 +19,214 @@ package p2p import ( + "fmt" "net" + "sync" + "sync/atomic" + "time" + "github.com/ethereum/go-ethereum/p2p/enode" + + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" ) -var ( - ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil) - ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil) - egressConnectMeter = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil) - egressTrafficMeter = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil) +const ( + MetricsInboundConnects = "p2p/InboundConnects" // Name for the registered inbound connects meter + MetricsInboundTraffic = "p2p/InboundTraffic" // Name for the registered inbound traffic meter + MetricsOutboundConnects = "p2p/OutboundConnects" // Name for the registered outbound connects meter + MetricsOutboundTraffic = "p2p/OutboundTraffic" // Name for the registered outbound traffic meter + + MeteredPeerLimit = 1024 // This amount of peers are individually metered ) +var ( + ingressConnectMeter = metrics.NewRegisteredMeter(MetricsInboundConnects, nil) // Meter counting the ingress connections + ingressTrafficMeter = metrics.NewRegisteredMeter(MetricsInboundTraffic, nil) // Meter metering the cumulative ingress traffic + egressConnectMeter = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections + egressTrafficMeter = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil) // Meter metering the cumulative egress traffic + + PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress + PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress + + meteredPeerFeed event.Feed // Event feed for peer metrics + meteredPeerCount int32 // Actually stored peer connection count +) + +// MeteredPeerEventType is the type of peer events emitted by a metered connection. +type MeteredPeerEventType int + +const ( + // PeerConnected is the type of event emitted when a peer successfully + // made the handshake. + PeerConnected MeteredPeerEventType = iota + + // PeerDisconnected is the type of event emitted when a peer disconnects. + PeerDisconnected + + // PeerHandshakeFailed is the type of event emitted when a peer fails to + // make the handshake or disconnects before the handshake. + PeerHandshakeFailed +) + +// MeteredPeerEvent is an event emitted when peers connect or disconnect. +type MeteredPeerEvent struct { + Type MeteredPeerEventType // Type of peer event + IP net.IP // IP address of the peer + ID enode.ID // NodeID of the peer + Elapsed time.Duration // Time elapsed between the connection and the handshake/disconnection + Ingress uint64 // Ingress count at the moment of the event + Egress uint64 // Egress count at the moment of the event +} + +// SubscribeMeteredPeerEvent registers a subscription for peer life-cycle events +// if metrics collection is enabled. +func SubscribeMeteredPeerEvent(ch chan<- MeteredPeerEvent) event.Subscription { + return meteredPeerFeed.Subscribe(ch) +} + // meteredConn is a wrapper around a net.Conn that meters both the // inbound and outbound network traffic. type meteredConn struct { net.Conn // Network connection to wrap with metering + + connected time.Time // Connection time of the peer + ip net.IP // IP address of the peer + id enode.ID // NodeID of the peer + + // trafficMetered denotes if the peer is registered in the traffic registries. + // Its value is true if the metered peer count doesn't reach the limit in the + // moment of the peer's connection. + trafficMetered bool + ingressMeter metrics.Meter // Meter for the read bytes of the peer + egressMeter metrics.Meter // Meter for the written bytes of the peer + + lock sync.RWMutex // Lock protecting the metered connection's internals } -// newMeteredConn creates a new metered connection, also bumping the ingress or -// egress connection meter. If the metrics system is disabled, this function -// returns the original object. -func newMeteredConn(conn net.Conn, ingress bool) net.Conn { +// newMeteredConn creates a new metered connection, bumps the ingress or egress +// connection meter and also increases the metered peer count. If the metrics +// system is disabled or the IP address is unspecified, this function returns +// the original object. +func newMeteredConn(conn net.Conn, ingress bool, ip net.IP) net.Conn { // Short circuit if metrics are disabled if !metrics.Enabled { return conn } - // Otherwise bump the connection counters and wrap the connection + if ip.IsUnspecified() { + log.Warn("Peer IP is unspecified") + return conn + } + // Bump the connection counters and wrap the connection if ingress { ingressConnectMeter.Mark(1) } else { egressConnectMeter.Mark(1) } - return &meteredConn{Conn: conn} + return &meteredConn{ + Conn: conn, + ip: ip, + connected: time.Now(), + } } -// Read delegates a network read to the underlying connection, bumping the ingress -// traffic meter along the way. +// Read delegates a network read to the underlying connection, bumping the common +// and the peer ingress traffic meters along the way. func (c *meteredConn) Read(b []byte) (n int, err error) { n, err = c.Conn.Read(b) ingressTrafficMeter.Mark(int64(n)) - return + c.lock.RLock() + if c.trafficMetered { + c.ingressMeter.Mark(int64(n)) + } + c.lock.RUnlock() + return n, err } -// Write delegates a network write to the underlying connection, bumping the -// egress traffic meter along the way. +// Write delegates a network write to the underlying connection, bumping the common +// and the peer egress traffic meters along the way. func (c *meteredConn) Write(b []byte) (n int, err error) { n, err = c.Conn.Write(b) egressTrafficMeter.Mark(int64(n)) - return + c.lock.RLock() + if c.trafficMetered { + c.egressMeter.Mark(int64(n)) + } + c.lock.RUnlock() + return n, err +} + +// handshakeDone is called when a peer handshake is done. Registers the peer to +// the ingress and the egress traffic registries using the peer's IP and node ID, +// also emits connect event. +func (c *meteredConn) handshakeDone(id enode.ID) { + if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit { + // Don't register the peer in the traffic registries. + atomic.AddInt32(&meteredPeerCount, -1) + c.lock.Lock() + c.id, c.trafficMetered = id, false + c.lock.Unlock() + log.Warn("Metered peer count reached the limit") + } else { + key := fmt.Sprintf("%s/%s", c.ip, id.String()) + c.lock.Lock() + c.id, c.trafficMetered = id, true + c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry) + c.egressMeter = metrics.NewRegisteredMeter(key, PeerEgressRegistry) + c.lock.Unlock() + } + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerConnected, + IP: c.ip, + ID: id, + Elapsed: time.Since(c.connected), + }) +} + +// Close delegates a close operation to the underlying connection, unregisters +// the peer from the traffic registries and emits close event. +func (c *meteredConn) Close() error { + err := c.Conn.Close() + c.lock.RLock() + if c.id == (enode.ID{}) { + // If the peer disconnects before the handshake. + c.lock.RUnlock() + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerHandshakeFailed, + IP: c.ip, + Elapsed: time.Since(c.connected), + }) + return err + } + id := c.id + if !c.trafficMetered { + // If the peer isn't registered in the traffic registries. + c.lock.RUnlock() + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerDisconnected, + IP: c.ip, + ID: id, + }) + return err + } + ingress, egress := uint64(c.ingressMeter.Count()), uint64(c.egressMeter.Count()) + c.lock.RUnlock() + + // Decrement the metered peer count + atomic.AddInt32(&meteredPeerCount, -1) + + // Unregister the peer from the traffic registries + key := fmt.Sprintf("%s/%s", c.ip, id) + PeerIngressRegistry.Unregister(key) + PeerEgressRegistry.Unregister(key) + + meteredPeerFeed.Send(MeteredPeerEvent{ + Type: PeerDisconnected, + IP: c.ip, + ID: id, + Ingress: ingress, + Egress: egress, + }) + return err } diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index a254648c66..8fad921c48 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -129,21 +129,15 @@ func Map(m Interface, c chan struct{}, protocol string, extport, intport int, na // ExtIP assumes that the local machine is reachable on the given // external IP address, and that any required ports were mapped manually. // Mapping operations will not return an error but won't actually do anything. -func ExtIP(ip net.IP) Interface { - if ip == nil { - panic("IP must not be nil") - } - return extIP(ip) -} +type ExtIP net.IP -type extIP net.IP - -func (n extIP) ExternalIP() (net.IP, error) { return net.IP(n), nil } -func (n extIP) String() string { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) } +func (n ExtIP) ExternalIP() (net.IP, error) { return net.IP(n), nil } +func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) } // These do nothing. -func (extIP) AddMapping(string, int, int, string, time.Duration) error { return nil } -func (extIP) DeleteMapping(string, int, int) error { return nil } + +func (ExtIP) AddMapping(string, int, int, string, time.Duration) error { return nil } +func (ExtIP) DeleteMapping(string, int, int) error { return nil } // Any returns a port mapper that tries to discover any supported // mechanism on the local network. diff --git a/p2p/nat/nat_test.go b/p2p/nat/nat_test.go index 469101e997..814e6d9e14 100644 --- a/p2p/nat/nat_test.go +++ b/p2p/nat/nat_test.go @@ -28,7 +28,7 @@ import ( func TestAutoDiscRace(t *testing.T) { ad := startautodisc("thing", func() Interface { time.Sleep(500 * time.Millisecond) - return extIP{33, 44, 55, 66} + return ExtIP{33, 44, 55, 66} }) // Spawn a few concurrent calls to ad.ExternalIP. diff --git a/p2p/netutil/iptrack.go b/p2p/netutil/iptrack.go new file mode 100644 index 0000000000..b9cbd5e1ca --- /dev/null +++ b/p2p/netutil/iptrack.go @@ -0,0 +1,130 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package netutil + +import ( + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +// IPTracker predicts the external endpoint, i.e. IP address and port, of the local host +// based on statements made by other hosts. +type IPTracker struct { + window time.Duration + contactWindow time.Duration + minStatements int + clock mclock.Clock + statements map[string]ipStatement + contact map[string]mclock.AbsTime + lastStatementGC mclock.AbsTime + lastContactGC mclock.AbsTime +} + +type ipStatement struct { + endpoint string + time mclock.AbsTime +} + +// NewIPTracker creates an IP tracker. +// +// The window parameters configure the amount of past network events which are kept. The +// minStatements parameter enforces a minimum number of statements which must be recorded +// before any prediction is made. Higher values for these parameters decrease 'flapping' of +// predictions as network conditions change. Window duration values should typically be in +// the range of minutes. +func NewIPTracker(window, contactWindow time.Duration, minStatements int) *IPTracker { + return &IPTracker{ + window: window, + contactWindow: contactWindow, + statements: make(map[string]ipStatement), + minStatements: minStatements, + contact: make(map[string]mclock.AbsTime), + clock: mclock.System{}, + } +} + +// PredictFullConeNAT checks whether the local host is behind full cone NAT. It predicts by +// checking whether any statement has been received from a node we didn't contact before +// the statement was made. +func (it *IPTracker) PredictFullConeNAT() bool { + now := it.clock.Now() + it.gcContact(now) + it.gcStatements(now) + for host, st := range it.statements { + if c, ok := it.contact[host]; !ok || c > st.time { + return true + } + } + return false +} + +// PredictEndpoint returns the current prediction of the external endpoint. +func (it *IPTracker) PredictEndpoint() string { + it.gcStatements(it.clock.Now()) + + // The current strategy is simple: find the endpoint with most statements. + counts := make(map[string]int) + maxcount, max := 0, "" + for _, s := range it.statements { + c := counts[s.endpoint] + 1 + counts[s.endpoint] = c + if c > maxcount && c >= it.minStatements { + maxcount, max = c, s.endpoint + } + } + return max +} + +// AddStatement records that a certain host thinks our external endpoint is the one given. +func (it *IPTracker) AddStatement(host, endpoint string) { + now := it.clock.Now() + it.statements[host] = ipStatement{endpoint, now} + if time.Duration(now-it.lastStatementGC) >= it.window { + it.gcStatements(now) + } +} + +// AddContact records that a packet containing our endpoint information has been sent to a +// certain host. +func (it *IPTracker) AddContact(host string) { + now := it.clock.Now() + it.contact[host] = now + if time.Duration(now-it.lastContactGC) >= it.contactWindow { + it.gcContact(now) + } +} + +func (it *IPTracker) gcStatements(now mclock.AbsTime) { + it.lastStatementGC = now + cutoff := now.Add(-it.window) + for host, s := range it.statements { + if s.time < cutoff { + delete(it.statements, host) + } + } +} + +func (it *IPTracker) gcContact(now mclock.AbsTime) { + it.lastContactGC = now + cutoff := now.Add(-it.contactWindow) + for host, ct := range it.contact { + if ct < cutoff { + delete(it.contact, host) + } + } +} diff --git a/p2p/netutil/iptrack_test.go b/p2p/netutil/iptrack_test.go new file mode 100644 index 0000000000..a9a2998a65 --- /dev/null +++ b/p2p/netutil/iptrack_test.go @@ -0,0 +1,138 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package netutil + +import ( + "fmt" + mrand "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +const ( + opStatement = iota + opContact + opPredict + opCheckFullCone +) + +type iptrackTestEvent struct { + op int + time int // absolute, in milliseconds + ip, from string +} + +func TestIPTracker(t *testing.T) { + tests := map[string][]iptrackTestEvent{ + "minStatements": { + {opPredict, 0, "", ""}, + {opStatement, 0, "127.0.0.1", "127.0.0.2"}, + {opPredict, 1000, "", ""}, + {opStatement, 1000, "127.0.0.1", "127.0.0.3"}, + {opPredict, 1000, "", ""}, + {opStatement, 1000, "127.0.0.1", "127.0.0.4"}, + {opPredict, 1000, "127.0.0.1", ""}, + }, + "window": { + {opStatement, 0, "127.0.0.1", "127.0.0.2"}, + {opStatement, 2000, "127.0.0.1", "127.0.0.3"}, + {opStatement, 3000, "127.0.0.1", "127.0.0.4"}, + {opPredict, 10000, "127.0.0.1", ""}, + {opPredict, 10001, "", ""}, // first statement expired + {opStatement, 10100, "127.0.0.1", "127.0.0.2"}, + {opPredict, 10200, "127.0.0.1", ""}, + }, + "fullcone": { + {opContact, 0, "", "127.0.0.2"}, + {opStatement, 10, "127.0.0.1", "127.0.0.2"}, + {opContact, 2000, "", "127.0.0.3"}, + {opStatement, 2010, "127.0.0.1", "127.0.0.3"}, + {opContact, 3000, "", "127.0.0.4"}, + {opStatement, 3010, "127.0.0.1", "127.0.0.4"}, + {opCheckFullCone, 3500, "false", ""}, + }, + "fullcone_2": { + {opContact, 0, "", "127.0.0.2"}, + {opStatement, 10, "127.0.0.1", "127.0.0.2"}, + {opContact, 2000, "", "127.0.0.3"}, + {opStatement, 2010, "127.0.0.1", "127.0.0.3"}, + {opStatement, 3000, "127.0.0.1", "127.0.0.4"}, + {opContact, 3010, "", "127.0.0.4"}, + {opCheckFullCone, 3500, "true", ""}, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { runIPTrackerTest(t, test) }) + } +} + +func runIPTrackerTest(t *testing.T, evs []iptrackTestEvent) { + var ( + clock mclock.Simulated + it = NewIPTracker(10*time.Second, 10*time.Second, 3) + ) + it.clock = &clock + for i, ev := range evs { + evtime := time.Duration(ev.time) * time.Millisecond + clock.Run(evtime - time.Duration(clock.Now())) + switch ev.op { + case opStatement: + it.AddStatement(ev.from, ev.ip) + case opContact: + it.AddContact(ev.from) + case opPredict: + if pred := it.PredictEndpoint(); pred != ev.ip { + t.Errorf("op %d: wrong prediction %q, want %q", i, pred, ev.ip) + } + case opCheckFullCone: + pred := fmt.Sprintf("%t", it.PredictFullConeNAT()) + if pred != ev.ip { + t.Errorf("op %d: wrong prediction %s, want %s", i, pred, ev.ip) + } + } + } +} + +// This checks that old statements and contacts are GCed even if Predict* isn't called. +func TestIPTrackerForceGC(t *testing.T) { + var ( + clock mclock.Simulated + window = 10 * time.Second + rate = 50 * time.Millisecond + max = int(window/rate) + 1 + it = NewIPTracker(window, window, 3) + ) + it.clock = &clock + + for i := 0; i < 5*max; i++ { + e1 := make([]byte, 4) + e2 := make([]byte, 4) + mrand.Read(e1) + mrand.Read(e2) + it.AddStatement(string(e1), string(e2)) + it.AddContact(string(e1)) + clock.Run(rate) + } + if len(it.contact) > 2*max { + t.Errorf("contacts not GCed, have %d", len(it.contact)) + } + if len(it.statements) > 2*max { + t.Errorf("statements not GCed, have %d", len(it.statements)) + } +} diff --git a/p2p/peer.go b/p2p/peer.go index b61cced549..af019d07a8 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -425,9 +425,10 @@ func (rw *protoRW) ReadMsg() (Msg, error) { // peer. Sub-protocol independent fields are contained and initialized here, with // protocol specifics delegated to all connected sub-protocols. type PeerInfo struct { - ID string `json:"id"` // Unique node identifier (also the encryption key) - Name string `json:"name"` // Name of the node, including client type, version, OS, custom data - Caps []string `json:"caps"` // Sum-protocols advertised by this particular peer + Enode string `json:"enode"` // Node URL + ID string `json:"id"` // Unique node identifier + Name string `json:"name"` // Name of the node, including client type, version, OS, custom data + Caps []string `json:"caps"` // Protocols advertised by this peer Network struct { LocalAddress string `json:"localAddress"` // Local endpoint of the TCP data connection RemoteAddress string `json:"remoteAddress"` // Remote endpoint of the TCP data connection @@ -447,6 +448,7 @@ func (p *Peer) Info() *PeerInfo { } // Assemble the generic peer metadata info := &PeerInfo{ + Enode: p.Node().String(), ID: p.ID().String(), Name: p.Name(), Caps: caps, diff --git a/p2p/protocol.go b/p2p/protocol.go index 4b90a2a708..9438ab8e47 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" ) // Protocol represents a P2P subprotocol implementation. @@ -52,6 +53,9 @@ type Protocol struct { // about a certain peer in the network. If an info retrieval function is set, // but returns nil, it is assumed that the protocol handshake is still running. PeerInfo func(id enode.ID) interface{} + + // Attributes contains protocol specific information for the node record. + Attributes []enr.Entry } func (p Protocol) cap() Cap { @@ -64,10 +68,6 @@ type Cap struct { Version uint } -func (cap Cap) RlpData() interface{} { - return []interface{}{cap.Name, cap.Version} -} - func (cap Cap) String() string { return fmt.Sprintf("%s/%d", cap.Name, cap.Version) } @@ -79,3 +79,5 @@ func (cs capsByNameAndVersion) Swap(i, j int) { cs[i], cs[j] = cs[j], cs[i] } func (cs capsByNameAndVersion) Less(i, j int) bool { return cs[i].Name < cs[j].Name || (cs[i].Name == cs[j].Name && cs[i].Version < cs[j].Version) } + +func (capsByNameAndVersion) ENRKey() string { return "cap" } diff --git a/p2p/protocols/accounting.go b/p2p/protocols/accounting.go new file mode 100644 index 0000000000..06a1a58454 --- /dev/null +++ b/p2p/protocols/accounting.go @@ -0,0 +1,172 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import "github.com/ethereum/go-ethereum/metrics" + +//define some metrics +var ( + //NOTE: these metrics just define the interfaces and are currently *NOT persisted* over sessions + //All metrics are cumulative + + //total amount of units credited + mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", nil) + //total amount of units debited + mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", nil) + //total amount of bytes credited + mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", nil) + //total amount of bytes debited + mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", nil) + //total amount of credited messages + mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", nil) + //total amount of debited messages + mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", nil) + //how many times local node had to drop remote peers + mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", nil) + //how many times local node overdrafted and dropped + mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", nil) +) + +//Prices defines how prices are being passed on to the accounting instance +type Prices interface { + //Return the Price for a message + Price(interface{}) *Price +} + +type Payer bool + +const ( + Sender = Payer(true) + Receiver = Payer(false) +) + +//Price represents the costs of a message +type Price struct { + Value uint64 // + PerByte bool //True if the price is per byte or for unit + Payer Payer +} + +//For gives back the price for a message +//A protocol provides the message price in absolute value +//This method then returns the correct signed amount, +//depending on who pays, which is identified by the `payer` argument: +//`Send` will pass a `Sender` payer, `Receive` will pass the `Receiver` argument. +//Thus: If Sending and sender pays, amount positive, otherwise negative +//If Receiving, and receiver pays, amount positive, otherwise negative +func (p *Price) For(payer Payer, size uint32) int64 { + price := p.Value + if p.PerByte { + price *= uint64(size) + } + if p.Payer == payer { + return 0 - int64(price) + } + return int64(price) +} + +//Balance is the actual accounting instance +//Balance defines the operations needed for accounting +//Implementations internally maintain the balance for every peer +type Balance interface { + //Adds amount to the local balance with remote node `peer`; + //positive amount = credit local node + //negative amount = debit local node + Add(amount int64, peer *Peer) error +} + +//Accounting implements the Hook interface +//It interfaces to the balances through the Balance interface, +//while interfacing with protocols and its prices through the Prices interface +type Accounting struct { + Balance //interface to accounting logic + Prices //interface to prices logic +} + +func NewAccounting(balance Balance, po Prices) *Accounting { + ah := &Accounting{ + Prices: po, + Balance: balance, + } + return ah +} + +//Implement Hook.Send +// Send takes a peer, a size and a msg and +// - calculates the cost for the local node sending a msg of size to peer using the Prices interface +// - credits/debits local node using balance interface +func (ah *Accounting) Send(peer *Peer, size uint32, msg interface{}) error { + //get the price for a message (through the protocol spec) + price := ah.Price(msg) + //this message doesn't need accounting + if price == nil { + return nil + } + //evaluate the price for sending messages + costToLocalNode := price.For(Sender, size) + //do the accounting + err := ah.Add(costToLocalNode, peer) + //record metrics: just increase counters for user-facing metrics + ah.doMetrics(costToLocalNode, size, err) + return err +} + +//Implement Hook.Receive +// Receive takes a peer, a size and a msg and +// - calculates the cost for the local node receiving a msg of size from peer using the Prices interface +// - credits/debits local node using balance interface +func (ah *Accounting) Receive(peer *Peer, size uint32, msg interface{}) error { + //get the price for a message (through the protocol spec) + price := ah.Price(msg) + //this message doesn't need accounting + if price == nil { + return nil + } + //evaluate the price for receiving messages + costToLocalNode := price.For(Receiver, size) + //do the accounting + err := ah.Add(costToLocalNode, peer) + //record metrics: just increase counters for user-facing metrics + ah.doMetrics(costToLocalNode, size, err) + return err +} + +//record some metrics +//this is not an error handling. `err` is returned by both `Send` and `Receive` +//`err` will only be non-nil if a limit has been violated (overdraft), in which case the peer has been dropped. +//if the limit has been violated and `err` is thus not nil: +// * if the price is positive, local node has been credited; thus `err` implicitly signals the REMOTE has been dropped +// * if the price is negative, local node has been debited, thus `err` implicitly signals LOCAL node "overdraft" +func (ah *Accounting) doMetrics(price int64, size uint32, err error) { + if price > 0 { + mBalanceCredit.Inc(price) + mBytesCredit.Inc(int64(size)) + mMsgCredit.Inc(1) + if err != nil { + //increase the number of times a remote node has been dropped due to "overdraft" + mPeerDrops.Inc(1) + } + } else { + mBalanceDebit.Inc(price) + mBytesDebit.Inc(int64(size)) + mMsgDebit.Inc(1) + if err != nil { + //increase the number of times the local node has done an "overdraft" in respect to other nodes + mSelfDrops.Inc(1) + } + } +} diff --git a/p2p/protocols/accounting_simulation_test.go b/p2p/protocols/accounting_simulation_test.go new file mode 100644 index 0000000000..65b737abed --- /dev/null +++ b/p2p/protocols/accounting_simulation_test.go @@ -0,0 +1,310 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "context" + "flag" + "fmt" + "math/rand" + "reflect" + "sync" + "testing" + "time" + + "github.com/mattn/go-colorable" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +const ( + content = "123456789" +) + +var ( + nodes = flag.Int("nodes", 30, "number of nodes to create (default 30)") + msgs = flag.Int("msgs", 100, "number of messages sent by node (default 100)") + loglevel = flag.Int("loglevel", 0, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") +) + +func init() { + flag.Parse() + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog)))) +} + +//TestAccountingSimulation runs a p2p/simulations simulation +//It creates a *nodes number of nodes, connects each one with each other, +//then sends out a random selection of messages up to *msgs amount of messages +//from the test protocol spec. +//The spec has some accounted messages defined through the Prices interface. +//The test does accounting for all the message exchanged, and then checks +//that every node has the same balance with a peer, but with opposite signs. +//Balance(AwithB) = 0 - Balance(BwithA) or Abs|Balance(AwithB)| == Abs|Balance(BwithA)| +func TestAccountingSimulation(t *testing.T) { + //setup the balances objects for every node + bal := newBalances(*nodes) + //define the node.Service for this test + services := adapters.Services{ + "accounting": func(ctx *adapters.ServiceContext) (node.Service, error) { + return bal.newNode(), nil + }, + } + //setup the simulation + adapter := adapters.NewSimAdapter(services) + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{DefaultService: "accounting"}) + defer net.Shutdown() + + // we send msgs messages per node, wait for all messages to arrive + bal.wg.Add(*nodes * *msgs) + trigger := make(chan enode.ID) + go func() { + // wait for all of them to arrive + bal.wg.Wait() + // then trigger a check + // the selected node for the trigger is irrelevant, + // we just want to trigger the end of the simulation + trigger <- net.Nodes[0].ID() + }() + + // create nodes and start them + for i := 0; i < *nodes; i++ { + conf := adapters.RandomNodeConfig() + bal.id2n[conf.ID] = i + if _, err := net.NewNodeWithConfig(conf); err != nil { + t.Fatal(err) + } + if err := net.Start(conf.ID); err != nil { + t.Fatal(err) + } + } + // fully connect nodes + for i, n := range net.Nodes { + for _, m := range net.Nodes[i+1:] { + if err := net.Connect(n.ID(), m.ID()); err != nil { + t.Fatal(err) + } + } + } + + // empty action + action := func(ctx context.Context) error { + return nil + } + // check always checks out + check := func(ctx context.Context, id enode.ID) (bool, error) { + return true, nil + } + + // run simulation + timeout := 30 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ + Action: action, + Trigger: trigger, + Expect: &simulations.Expectation{ + Nodes: []enode.ID{net.Nodes[0].ID()}, + Check: check, + }, + }) + + if result.Error != nil { + t.Fatal(result.Error) + } + + // check if balance matrix is symmetric + if err := bal.symmetric(); err != nil { + t.Fatal(err) + } +} + +// matrix is a matrix of nodes and its balances +// matrix is in fact a linear array of size n*n, +// so the balance for any node A with B is at index +// A*n + B, while the balance of node B with A is at +// B*n + A +// (n entries in the array will not be filled - +// the balance of a node with itself) +type matrix struct { + n int //number of nodes + m []int64 //array of balances +} + +// create a new matrix +func newMatrix(n int) *matrix { + return &matrix{ + n: n, + m: make([]int64, n*n), + } +} + +// called from the testBalance's Add accounting function: register balance change +func (m *matrix) add(i, j int, v int64) error { + // index for the balance of local node i with remote nodde j is + // i * number of nodes + remote node + mi := i*m.n + j + // register that balance + m.m[mi] += v + return nil +} + +// check that the balances are symmetric: +// balance of node i with node j is the same as j with i but with inverted signs +func (m *matrix) symmetric() error { + //iterate all nodes + for i := 0; i < m.n; i++ { + //iterate starting +1 + for j := i + 1; j < m.n; j++ { + log.Debug("bal", "1", i, "2", j, "i,j", m.m[i*m.n+j], "j,i", m.m[j*m.n+i]) + if m.m[i*m.n+j] != -m.m[j*m.n+i] { + return fmt.Errorf("value mismatch. m[%v, %v] = %v; m[%v, %v] = %v", i, j, m.m[i*m.n+j], j, i, m.m[j*m.n+i]) + } + } + } + return nil +} + +// all the balances +type balances struct { + i int + *matrix + id2n map[enode.ID]int + wg *sync.WaitGroup +} + +func newBalances(n int) *balances { + return &balances{ + matrix: newMatrix(n), + id2n: make(map[enode.ID]int), + wg: &sync.WaitGroup{}, + } +} + +// create a new testNode for every node created as part of the service +func (b *balances) newNode() *testNode { + defer func() { b.i++ }() + return &testNode{ + bal: b, + i: b.i, + peers: make([]*testPeer, b.n), //a node will be connected to n-1 peers + } +} + +type testNode struct { + bal *balances + i int + lock sync.Mutex + peers []*testPeer + peerCount int +} + +// do the accounting for the peer's test protocol +// testNode implements protocols.Balance +func (t *testNode) Add(a int64, p *Peer) error { + //get the index for the remote peer + remote := t.bal.id2n[p.ID()] + log.Debug("add", "local", t.i, "remote", remote, "amount", a) + return t.bal.add(t.i, remote, a) +} + +//run the p2p protocol +//for every node, represented by testNode, create a remote testPeer +func (t *testNode) run(p *p2p.Peer, rw p2p.MsgReadWriter) error { + spec := createTestSpec() + //create accounting hook + spec.Hook = NewAccounting(t, &dummyPrices{}) + + //create a peer for this node + tp := &testPeer{NewPeer(p, rw, spec), t.i, t.bal.id2n[p.ID()], t.bal.wg} + t.lock.Lock() + t.peers[t.bal.id2n[p.ID()]] = tp + t.peerCount++ + if t.peerCount == t.bal.n-1 { + //when all peer connections are established, start sending messages from this peer + go t.send() + } + t.lock.Unlock() + return tp.Run(tp.handle) +} + +// p2p message receive handler function +func (tp *testPeer) handle(ctx context.Context, msg interface{}) error { + tp.wg.Done() + log.Debug("receive", "from", tp.remote, "to", tp.local, "type", reflect.TypeOf(msg), "msg", msg) + return nil +} + +type testPeer struct { + *Peer + local, remote int + wg *sync.WaitGroup +} + +func (t *testNode) send() { + log.Debug("start sending") + for i := 0; i < *msgs; i++ { + //determine randomly to which peer to send + whom := rand.Intn(t.bal.n - 1) + if whom >= t.i { + whom++ + } + t.lock.Lock() + p := t.peers[whom] + t.lock.Unlock() + + //determine a random message from the spec's messages to be sent + which := rand.Intn(len(p.spec.Messages)) + msg := p.spec.Messages[which] + switch msg.(type) { + case *perBytesMsgReceiverPays: + msg = &perBytesMsgReceiverPays{Content: content[:rand.Intn(len(content))]} + case *perBytesMsgSenderPays: + msg = &perBytesMsgSenderPays{Content: content[:rand.Intn(len(content))]} + } + log.Debug("send", "from", t.i, "to", whom, "type", reflect.TypeOf(msg), "msg", msg) + p.Send(context.TODO(), msg) + } +} + +// define the protocol +func (t *testNode) Protocols() []p2p.Protocol { + return []p2p.Protocol{{ + Length: 100, + Run: t.run, + }} +} + +func (t *testNode) APIs() []rpc.API { + return nil +} + +func (t *testNode) Start(server *p2p.Server) error { + return nil +} + +func (t *testNode) Stop() error { + return nil +} diff --git a/p2p/protocols/accounting_test.go b/p2p/protocols/accounting_test.go new file mode 100644 index 0000000000..3810ae2c9b --- /dev/null +++ b/p2p/protocols/accounting_test.go @@ -0,0 +1,223 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "testing" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" +) + +//dummy Balance implementation +type dummyBalance struct { + amount int64 + peer *Peer +} + +//dummy Prices implementation +type dummyPrices struct{} + +//a dummy message which needs size based accounting +//sender pays +type perBytesMsgSenderPays struct { + Content string +} + +//a dummy message which needs size based accounting +//receiver pays +type perBytesMsgReceiverPays struct { + Content string +} + +//a dummy message which is paid for per unit +//sender pays +type perUnitMsgSenderPays struct{} + +//receiver pays +type perUnitMsgReceiverPays struct{} + +//a dummy message which has zero as its price +type zeroPriceMsg struct{} + +//a dummy message which has no accounting +type nilPriceMsg struct{} + +//return the price for the defined messages +func (d *dummyPrices) Price(msg interface{}) *Price { + switch msg.(type) { + //size based message cost, receiver pays + case *perBytesMsgReceiverPays: + return &Price{ + PerByte: true, + Value: uint64(100), + Payer: Receiver, + } + //size based message cost, sender pays + case *perBytesMsgSenderPays: + return &Price{ + PerByte: true, + Value: uint64(100), + Payer: Sender, + } + //unitary cost, receiver pays + case *perUnitMsgReceiverPays: + return &Price{ + PerByte: false, + Value: uint64(99), + Payer: Receiver, + } + //unitary cost, sender pays + case *perUnitMsgSenderPays: + return &Price{ + PerByte: false, + Value: uint64(99), + Payer: Sender, + } + case *zeroPriceMsg: + return &Price{ + PerByte: false, + Value: uint64(0), + Payer: Sender, + } + case *nilPriceMsg: + return nil + } + return nil +} + +//dummy accounting implementation, only stores values for later check +func (d *dummyBalance) Add(amount int64, peer *Peer) error { + d.amount = amount + d.peer = peer + return nil +} + +type testCase struct { + msg interface{} + size uint32 + sendResult int64 + recvResult int64 +} + +//lowest level unit test +func TestBalance(t *testing.T) { + //create instances + balance := &dummyBalance{} + prices := &dummyPrices{} + //create the spec + spec := createTestSpec() + //create the accounting hook for the spec + acc := NewAccounting(balance, prices) + //create a peer + id := adapters.RandomNodeConfig().ID + p := p2p.NewPeer(id, "testPeer", nil) + peer := NewPeer(p, &dummyRW{}, spec) + //price depends on size, receiver pays + msg := &perBytesMsgReceiverPays{Content: "testBalance"} + size, _ := rlp.EncodeToBytes(msg) + + testCases := []testCase{ + { + msg, + uint32(len(size)), + int64(len(size) * 100), + int64(len(size) * -100), + }, + { + &perBytesMsgSenderPays{Content: "testBalance"}, + uint32(len(size)), + int64(len(size) * -100), + int64(len(size) * 100), + }, + { + &perUnitMsgSenderPays{}, + 0, + int64(-99), + int64(99), + }, + { + &perUnitMsgReceiverPays{}, + 0, + int64(99), + int64(-99), + }, + { + &zeroPriceMsg{}, + 0, + int64(0), + int64(0), + }, + { + &nilPriceMsg{}, + 0, + int64(0), + int64(0), + }, + } + checkAccountingTestCases(t, testCases, acc, peer, balance, true) + checkAccountingTestCases(t, testCases, acc, peer, balance, false) +} + +func checkAccountingTestCases(t *testing.T, cases []testCase, acc *Accounting, peer *Peer, balance *dummyBalance, send bool) { + for _, c := range cases { + var err error + var expectedResult int64 + //reset balance before every check + balance.amount = 0 + if send { + err = acc.Send(peer, c.size, c.msg) + expectedResult = c.sendResult + } else { + err = acc.Receive(peer, c.size, c.msg) + expectedResult = c.recvResult + } + + checkResults(t, err, balance, peer, expectedResult) + } +} + +func checkResults(t *testing.T, err error, balance *dummyBalance, peer *Peer, result int64) { + if err != nil { + t.Fatal(err) + } + if balance.peer != peer { + t.Fatalf("expected Add to be called with peer %v, got %v", peer, balance.peer) + } + if balance.amount != result { + t.Fatalf("Expected balance to be %d but is %d", result, balance.amount) + } +} + +//create a test spec +func createTestSpec() *Spec { + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + &perBytesMsgReceiverPays{}, + &perBytesMsgSenderPays{}, + &perUnitMsgReceiverPays{}, + &perUnitMsgSenderPays{}, + &zeroPriceMsg{}, + &nilPriceMsg{}, + }, + } + return spec +} diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index 615f74b569..7dddd852fa 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -122,6 +122,16 @@ type WrappedMsg struct { Payload []byte } +//For accounting, the design is to allow the Spec to describe which and how its messages are priced +//To access this functionality, we provide a Hook interface which will call accounting methods +//NOTE: there could be more such (horizontal) hooks in the future +type Hook interface { + //A hook for sending messages + Send(peer *Peer, size uint32, msg interface{}) error + //A hook for receiving messages + Receive(peer *Peer, size uint32, msg interface{}) error +} + // Spec is a protocol specification including its name and version as well as // the types of messages which are exchanged type Spec struct { @@ -141,6 +151,9 @@ type Spec struct { // each message must have a single unique data type Messages []interface{} + //hook for accounting (could be extended to multiple hooks in the future) + Hook Hook + initOnce sync.Once codes map[reflect.Type]uint64 types map[uint64]reflect.Type @@ -274,6 +287,15 @@ func (p *Peer) Send(ctx context.Context, msg interface{}) error { Payload: r, } + //if the accounting hook is set, call it + if p.spec.Hook != nil { + err := p.spec.Hook.Send(p, wmsg.Size, msg) + if err != nil { + p.Drop(err) + return err + } + } + code, found := p.spec.GetCode(msg) if !found { return errorf(ErrInvalidMsgType, "%v", code) @@ -336,6 +358,14 @@ func (p *Peer) handleIncoming(handle func(ctx context.Context, msg interface{}) return errorf(ErrDecode, "<= %v: %v", msg, err) } + //if the accounting hook is set, call it + if p.spec.Hook != nil { + err := p.spec.Hook.Receive(p, wmsg.Size, val) + if err != nil { + return err + } + } + // call the registered handler callbacks // a registered callback take the decoded message as argument as an interface // which the handler is supposed to cast to the appropriate type diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index 4755db3e62..a26222cd8b 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -17,12 +17,15 @@ package protocols import ( + "bytes" "context" "errors" "fmt" "testing" "time" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" @@ -185,6 +188,169 @@ func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) { } } +type dummyHook struct { + peer *Peer + size uint32 + msg interface{} + send bool + err error + waitC chan struct{} +} + +type dummyMsg struct { + Content string +} + +func (d *dummyHook) Send(peer *Peer, size uint32, msg interface{}) error { + d.peer = peer + d.size = size + d.msg = msg + d.send = true + return d.err +} + +func (d *dummyHook) Receive(peer *Peer, size uint32, msg interface{}) error { + d.peer = peer + d.size = size + d.msg = msg + d.send = false + d.waitC <- struct{}{} + return d.err +} + +func TestProtocolHook(t *testing.T) { + testHook := &dummyHook{ + waitC: make(chan struct{}, 1), + } + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + dummyMsg{}, + }, + Hook: testHook, + } + + runFunc := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := NewPeer(p, rw, spec) + ctx := context.TODO() + err := peer.Send(ctx, &dummyMsg{ + Content: "handshake"}) + + if err != nil { + t.Fatal(err) + } + + handle := func(ctx context.Context, msg interface{}) error { + return nil + } + + return peer.Run(handle) + } + + conf := adapters.RandomNodeConfig() + tester := p2ptest.NewProtocolTester(t, conf.ID, 2, runFunc) + err := tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { + Code: 0, + Msg: &dummyMsg{Content: "handshake"}, + Peer: tester.Nodes[0].ID(), + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if testHook.msg == nil || testHook.msg.(*dummyMsg).Content != "handshake" { + t.Fatal("Expected msg to be set, but it is not") + } + if !testHook.send { + t.Fatal("Expected a send message, but it is not") + } + if testHook.peer == nil || testHook.peer.ID() != tester.Nodes[0].ID() { + t.Fatal("Expected peer ID to be set correctly, but it is not") + } + if testHook.size != 11 { //11 is the length of the encoded message + t.Fatalf("Expected size to be %d, but it is %d ", 1, testHook.size) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &dummyMsg{Content: "response"}, + Peer: tester.Nodes[1].ID(), + }, + }, + }) + + <-testHook.waitC + + if err != nil { + t.Fatal(err) + } + if testHook.msg == nil || testHook.msg.(*dummyMsg).Content != "response" { + t.Fatal("Expected msg to be set, but it is not") + } + if testHook.send { + t.Fatal("Expected a send message, but it is not") + } + if testHook.peer == nil || testHook.peer.ID() != tester.Nodes[1].ID() { + t.Fatal("Expected peer ID to be set correctly, but it is not") + } + if testHook.size != 10 { //11 is the length of the encoded message + t.Fatalf("Expected size to be %d, but it is %d ", 1, testHook.size) + } + + testHook.err = fmt.Errorf("dummy error") + err = tester.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &dummyMsg{Content: "response"}, + Peer: tester.Nodes[1].ID(), + }, + }, + }) + + <-testHook.waitC + + time.Sleep(100 * time.Millisecond) + err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: tester.Nodes[1].ID(), Error: testHook.err}) + if err != nil { + t.Fatalf("Expected a specific disconnect error, but got different one: %v", err) + } + +} + +//We need to test that if the hook is not defined, then message infrastructure +//(send,receive) still works +func TestNoHook(t *testing.T) { + //create a test spec + spec := createTestSpec() + //a random node + id := adapters.RandomNodeConfig().ID + //a peer + p := p2p.NewPeer(id, "testPeer", nil) + rw := &dummyRW{} + peer := NewPeer(p, rw, spec) + ctx := context.TODO() + msg := &perBytesMsgSenderPays{Content: "testBalance"} + //send a message + err := peer.Send(ctx, msg) + if err != nil { + t.Fatal(err) + } + //simulate receiving a message + rw.msg = msg + peer.handleIncoming(func(ctx context.Context, msg interface{}) error { + return nil + }) + //all should just work and not result in any error +} + func TestProtoHandshakeVersionMismatch(t *testing.T) { runProtoHandshake(t, &protoHandshake{41, "420"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 41 (!= 42)").Error())) } @@ -386,3 +552,39 @@ func XTestMultiplePeersDropOther(t *testing.T) { fmt.Errorf("subprotocol error"), ) } + +//dummy implementation of a MsgReadWriter +//this allows for quick and easy unit tests without +//having to build up the complete protocol +type dummyRW struct { + msg interface{} + size uint32 + code uint64 +} + +func (d *dummyRW) WriteMsg(msg p2p.Msg) error { + return nil +} + +func (d *dummyRW) ReadMsg() (p2p.Msg, error) { + enc := bytes.NewReader(d.getDummyMsg()) + return p2p.Msg{ + Code: d.code, + Size: d.size, + Payload: enc, + ReceivedAt: time.Now(), + }, nil +} + +func (d *dummyRW) getDummyMsg() []byte { + r, _ := rlp.EncodeToBytes(d.msg) + var b bytes.Buffer + wmsg := WrappedMsg{ + Context: b.Bytes(), + Size: uint32(len(r)), + Payload: r, + } + rr, _ := rlp.EncodeToBytes(wmsg) + d.size = uint32(len(rr)) + return rr +} diff --git a/p2p/rlpx.go b/p2p/rlpx.go index a105720a44..22a27dd96a 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -151,7 +151,7 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake, } if msg.Code == discMsg { // Disconnect before protocol handshake is valid according to the - // spec and we send it ourself if the posthanshake checks fail. + // spec and we send it ourself if the post-handshake checks fail. // We can't return the reason directly, though, because it is echoed // back otherwise. Wrap it in a string instead. var reason [1]DiscReason diff --git a/p2p/server.go b/p2p/server.go index 40db758e2d..6678608634 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -20,9 +20,11 @@ package p2p import ( "bytes" "crypto/ecdsa" + "encoding/hex" "errors" "fmt" "net" + "sort" "sync" "sync/atomic" "time" @@ -35,8 +37,10 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" + "github.com/ethereum/go-ethereum/rlp" ) const ( @@ -160,6 +164,8 @@ type Server struct { lock sync.Mutex // protects running running bool + nodedb *enode.DB + localnode *enode.LocalNode ntab discoverTable listener net.Listener ourHandshake *protoHandshake @@ -222,7 +228,7 @@ type transport interface { MsgReadWriter // transports must provide Close because we use MsgPipe in some of // the tests. Closing the actual network connection doesn't do - // anything in those tests because NsgPipe doesn't use it. + // anything in those tests because MsgPipe doesn't use it. close(err error) } @@ -347,43 +353,13 @@ func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription { // Self returns the local node's endpoint information. func (srv *Server) Self() *enode.Node { srv.lock.Lock() - running, listener, ntab := srv.running, srv.listener, srv.ntab + ln := srv.localnode srv.lock.Unlock() - if !running { + if ln == nil { return enode.NewV4(&srv.PrivateKey.PublicKey, net.ParseIP("0.0.0.0"), 0, 0) } - return srv.makeSelf(listener, ntab) -} - -func (srv *Server) makeSelf(listener net.Listener, ntab discoverTable) *enode.Node { - // If the node is running but discovery is off, manually assemble the node infos. - if ntab == nil { - addr := srv.tcpAddr(listener) - return enode.NewV4(&srv.PrivateKey.PublicKey, addr.IP, addr.Port, 0) - } - // Otherwise return the discovery node. - return ntab.Self() -} - -func (srv *Server) tcpAddr(listener net.Listener) net.TCPAddr { - addr := net.TCPAddr{IP: net.IP{0, 0, 0, 0}} - if listener == nil { - return addr // Inbound connections disabled, use zero address. - } - // Otherwise inject the listener address too. - if a, ok := listener.Addr().(*net.TCPAddr); ok { - addr = *a - } - if srv.NAT != nil { - if ip, err := srv.NAT.ExternalIP(); err == nil { - addr.IP = ip - } - } - if addr.IP.IsUnspecified() { - addr.IP = net.IP{127, 0, 0, 1} - } - return addr + return ln.Node() } // Stop terminates the server and all active peer connections. @@ -443,7 +419,9 @@ func (srv *Server) Start() (err error) { if srv.log == nil { srv.log = log.New() } - srv.log.Info("Starting P2P networking") + if srv.NoDial && srv.ListenAddr == "" { + srv.log.Warn("P2P server will be useless, neither dialing nor listening") + } // static fields if srv.PrivateKey == nil { @@ -466,65 +444,120 @@ func (srv *Server) Start() (err error) { srv.peerOp = make(chan peerOpFunc) srv.peerOpDone = make(chan struct{}) - var ( - conn *net.UDPConn - sconn *sharedUDPConn - realaddr *net.UDPAddr - unhandled chan discover.ReadPacket - ) - - if !srv.NoDiscovery || srv.DiscoveryV5 { - addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) - if err != nil { + if err := srv.setupLocalNode(); err != nil { + return err + } + if srv.ListenAddr != "" { + if err := srv.setupListening(); err != nil { return err } - conn, err = net.ListenUDP("udp", addr) - if err != nil { - return err - } - realaddr = conn.LocalAddr().(*net.UDPAddr) - if srv.NAT != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - // TODO: react to external IP changes over time. - if ext, err := srv.NAT.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} - } - } + } + if err := srv.setupDiscovery(); err != nil { + return err } - if !srv.NoDiscovery && srv.DiscoveryV5 { - unhandled = make(chan discover.ReadPacket, 100) - sconn = &sharedUDPConn{conn, unhandled} + dynPeers := srv.maxDialedConns() + dialer := newDialState(srv.localnode.ID(), srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) + srv.loopWG.Add(1) + go srv.run(dialer) + return nil +} + +func (srv *Server) setupLocalNode() error { + // Create the devp2p handshake. + pubkey := crypto.FromECDSAPub(&srv.PrivateKey.PublicKey) + srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: pubkey[1:]} + for _, p := range srv.Protocols { + srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) + } + sort.Sort(capsByNameAndVersion(srv.ourHandshake.Caps)) + + // Create the local node. + db, err := enode.OpenDB(srv.Config.NodeDatabase) + if err != nil { + return err + } + srv.nodedb = db + srv.localnode = enode.NewLocalNode(db, srv.PrivateKey) + srv.localnode.SetFallbackIP(net.IP{127, 0, 0, 1}) + srv.localnode.Set(capsByNameAndVersion(srv.ourHandshake.Caps)) + // TODO: check conflicts + for _, p := range srv.Protocols { + for _, e := range p.Attributes { + srv.localnode.Set(e) + } + } + switch srv.NAT.(type) { + case nil: + // No NAT interface, do nothing. + case nat.ExtIP: + // ExtIP doesn't block, set the IP right away. + ip, _ := srv.NAT.ExternalIP() + srv.localnode.SetStaticIP(ip) + default: + // Ask the router about the IP. This takes a while and blocks startup, + // do it in the background. + srv.loopWG.Add(1) + go func() { + defer srv.loopWG.Done() + if ip, err := srv.NAT.ExternalIP(); err == nil { + srv.localnode.SetStaticIP(ip) + } + }() + } + return nil +} + +func (srv *Server) setupDiscovery() error { + if srv.NoDiscovery && !srv.DiscoveryV5 { + return nil } - // node table + addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) + if err != nil { + return err + } + conn, err := net.ListenUDP("udp", addr) + if err != nil { + return err + } + realaddr := conn.LocalAddr().(*net.UDPAddr) + srv.log.Debug("UDP listener up", "addr", realaddr) + if srv.NAT != nil { + if !realaddr.IP.IsLoopback() { + go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + } + } + srv.localnode.SetFallbackUDP(realaddr.Port) + + // Discovery V4 + var unhandled chan discover.ReadPacket + var sconn *sharedUDPConn if !srv.NoDiscovery { - cfg := discover.Config{ - PrivateKey: srv.PrivateKey, - AnnounceAddr: realaddr, - NodeDBPath: srv.NodeDatabase, - NetRestrict: srv.NetRestrict, - Bootnodes: srv.BootstrapNodes, - Unhandled: unhandled, + if srv.DiscoveryV5 { + unhandled = make(chan discover.ReadPacket, 100) + sconn = &sharedUDPConn{conn, unhandled} } - ntab, err := discover.ListenUDP(conn, cfg) + cfg := discover.Config{ + PrivateKey: srv.PrivateKey, + NetRestrict: srv.NetRestrict, + Bootnodes: srv.BootstrapNodes, + Unhandled: unhandled, + } + ntab, err := discover.ListenUDP(conn, srv.localnode, cfg) if err != nil { return err } srv.ntab = ntab } - + // Discovery V5 if srv.DiscoveryV5 { - var ( - ntab *discv5.Network - err error - ) + var ntab *discv5.Network + var err error if sconn != nil { - ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, "", srv.NetRestrict) } else { - ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, "", srv.NetRestrict) } if err != nil { return err @@ -534,32 +567,10 @@ func (srv *Server) Start() (err error) { } srv.DiscV5 = ntab } - - dynPeers := srv.maxDialedConns() - dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) - - // handshake - pubkey := crypto.FromECDSAPub(&srv.PrivateKey.PublicKey) - srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: pubkey[1:]} - for _, p := range srv.Protocols { - srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) - } - // listen/dial - if srv.ListenAddr != "" { - if err := srv.startListening(); err != nil { - return err - } - } - if srv.NoDial && srv.ListenAddr == "" { - srv.log.Warn("P2P server will be useless, neither dialing nor listening") - } - - srv.loopWG.Add(1) - go srv.run(dialer) return nil } -func (srv *Server) startListening() error { +func (srv *Server) setupListening() error { // Launch the TCP listener. listener, err := net.Listen("tcp", srv.ListenAddr) if err != nil { @@ -568,8 +579,11 @@ func (srv *Server) startListening() error { laddr := listener.Addr().(*net.TCPAddr) srv.ListenAddr = laddr.String() srv.listener = listener + srv.localnode.Set(enr.TCP(laddr.Port)) + srv.loopWG.Add(1) go srv.listenLoop() + // Map the TCP listening port if NAT is configured. if !laddr.IP.IsLoopback() && srv.NAT != nil { srv.loopWG.Add(1) @@ -589,7 +603,10 @@ type dialer interface { } func (srv *Server) run(dialstate dialer) { + srv.log.Info("Started P2P networking", "self", srv.localnode.Node()) defer srv.loopWG.Done() + defer srv.nodedb.Close() + var ( peers = make(map[enode.ID]*Peer) inboundCount = 0 @@ -781,7 +798,7 @@ func (srv *Server) encHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int return DiscTooManyPeers case peers[c.node.ID()] != nil: return DiscAlreadyConnected - case c.node.ID() == srv.Self().ID(): + case c.node.ID() == srv.localnode.ID(): return DiscSelf default: return nil @@ -802,15 +819,11 @@ func (srv *Server) maxDialedConns() int { return srv.MaxPeers / r } -type tempError interface { - Temporary() bool -} - // listenLoop runs in its own goroutine and accepts // inbound connections. func (srv *Server) listenLoop() { defer srv.loopWG.Done() - srv.log.Info("RLPx listener up", "self", srv.Self()) + srv.log.Debug("TCP listener up", "addr", srv.listener.Addr()) tokens := defaultMaxPendingPeers if srv.MaxPendingPeers > 0 { @@ -831,7 +844,7 @@ func (srv *Server) listenLoop() { ) for { fd, err = srv.listener.Accept() - if tempErr, ok := err.(tempError); ok && tempErr.Temporary() { + if netutil.IsTemporaryError(err) { srv.log.Debug("Temporary read error", "err", err) continue } else if err != nil { @@ -851,7 +864,11 @@ func (srv *Server) listenLoop() { } } - fd = newMeteredConn(fd, true) + var ip net.IP + if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok { + ip = tcp.IP + } + fd = newMeteredConn(fd, true, ip) srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) go func() { srv.SetupConn(fd, inboundConn, nil) @@ -864,10 +881,6 @@ func (srv *Server) listenLoop() { // as a peer. It returns when the connection has been added as a peer // or the handshakes have failed. func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { - self := srv.Self() - if self == nil { - return errors.New("shutdown") - } c := &conn{fd: fd, transport: srv.newTransport(fd), flags: flags, cont: make(chan error)} err := srv.setupConn(c, flags, dialDest) if err != nil { @@ -908,6 +921,9 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro } else { c.node = nodeFromConn(remotePubkey, c.fd) } + if conn, ok := c.fd.(*meteredConn); ok { + conn.handshakeDone(c.node.ID()) + } clog := srv.log.New("id", c.node.ID(), "addr", c.fd.RemoteAddr(), "conn", c.flags) err = srv.checkpoint(c, srv.posthandshake) if err != nil { @@ -1003,6 +1019,7 @@ type NodeInfo struct { ID string `json:"id"` // Unique node identifier (also the encryption key) Name string `json:"name"` // Name of the node, including client type, version, OS, custom data Enode string `json:"enode"` // Enode URL for adding this peer from remote peers + ENR string `json:"enr"` // Ethereum Node Record IP string `json:"ip"` // IP address of the node Ports struct { Discovery int `json:"discovery"` // UDP listening port for discovery protocol @@ -1014,9 +1031,8 @@ type NodeInfo struct { // NodeInfo gathers and returns a collection of metadata known about the host. func (srv *Server) NodeInfo() *NodeInfo { - node := srv.Self() - // Gather and assemble the generic node infos + node := srv.Self() info := &NodeInfo{ Name: srv.Name, Enode: node.String(), @@ -1027,6 +1043,9 @@ func (srv *Server) NodeInfo() *NodeInfo { } info.Ports.Discovery = node.UDP() info.Ports.Listener = node.TCP() + if enc, err := rlp.EncodeToBytes(node.Record()); err == nil { + info.ENR = "0x" + hex.EncodeToString(enc) + } // Gather all the running protocol infos (only once per protocol type) for _, proto := range srv.Protocols { diff --git a/p2p/server_test.go b/p2p/server_test.go index e0b1fc122e..7e11577d6c 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -225,12 +225,15 @@ func TestServerTaskScheduling(t *testing.T) { // The Server in this test isn't actually running // because we're only interested in what run does. + db, _ := enode.OpenDB("") srv := &Server{ - Config: Config{MaxPeers: 10}, - quit: make(chan struct{}), - ntab: fakeTable{}, - running: true, - log: log.New(), + Config: Config{MaxPeers: 10}, + localnode: enode.NewLocalNode(db, newkey()), + nodedb: db, + quit: make(chan struct{}), + ntab: fakeTable{}, + running: true, + log: log.New(), } srv.loopWG.Add(1) go func() { @@ -271,11 +274,14 @@ func TestServerManyTasks(t *testing.T) { } var ( - srv = &Server{ - quit: make(chan struct{}), - ntab: fakeTable{}, - running: true, - log: log.New(), + db, _ = enode.OpenDB("") + srv = &Server{ + quit: make(chan struct{}), + localnode: enode.NewLocalNode(db, newkey()), + nodedb: db, + ntab: fakeTable{}, + running: true, + log: log.New(), } done = make(chan *testTask) start, end = 0, 0 diff --git a/p2p/simulations/README.md b/p2p/simulations/README.md index d1f8649eae..871d71b2c7 100644 --- a/p2p/simulations/README.md +++ b/p2p/simulations/README.md @@ -63,18 +63,6 @@ using the devp2p node stack rather than executing `main()`. The nodes listen for devp2p connections and WebSocket RPC clients on random localhost ports. -### DockerAdapter - -The `DockerAdapter` is similar to the `ExecAdapter` but executes `docker run` -to run the node in a Docker container using a Docker image containing the -simulation binary at `/bin/p2p-node`. - -The Docker image is built using `docker build` when the adapter is initialised, -meaning no prior setup is necessary other than having a working Docker client. - -Each node listens on the external IP of the container and the default p2p and -RPC ports (`30303` and `8546` respectively). - ## Network A simulation network is created with an ID and default service (which is used diff --git a/p2p/simulations/adapters/docker.go b/p2p/simulations/adapters/docker.go deleted file mode 100644 index 82eab0e9cd..0000000000 --- a/p2p/simulations/adapters/docker.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package adapters - -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - - "github.com/docker/docker/pkg/reexec" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/enode" -) - -var ( - ErrLinuxOnly = errors.New("DockerAdapter can only be used on Linux as it uses the current binary (which must be a Linux binary)") -) - -// DockerAdapter is a NodeAdapter which runs simulation nodes inside Docker -// containers. -// -// A Docker image is built which contains the current binary at /bin/p2p-node -// which when executed runs the underlying service (see the description -// of the execP2PNode function for more details) -type DockerAdapter struct { - ExecAdapter -} - -// NewDockerAdapter builds the p2p-node Docker image containing the current -// binary and returns a DockerAdapter -func NewDockerAdapter() (*DockerAdapter, error) { - // Since Docker containers run on Linux and this adapter runs the - // current binary in the container, it must be compiled for Linux. - // - // It is reasonable to require this because the caller can just - // compile the current binary in a Docker container. - if runtime.GOOS != "linux" { - return nil, ErrLinuxOnly - } - - if err := buildDockerImage(); err != nil { - return nil, err - } - - return &DockerAdapter{ - ExecAdapter{ - nodes: make(map[enode.ID]*ExecNode), - }, - }, nil -} - -// Name returns the name of the adapter for logging purposes -func (d *DockerAdapter) Name() string { - return "docker-adapter" -} - -// NewNode returns a new DockerNode using the given config -func (d *DockerAdapter) NewNode(config *NodeConfig) (Node, error) { - if len(config.Services) == 0 { - return nil, errors.New("node must have at least one service") - } - for _, service := range config.Services { - if _, exists := serviceFuncs[service]; !exists { - return nil, fmt.Errorf("unknown node service %q", service) - } - } - - // generate the config - conf := &execNodeConfig{ - Stack: node.DefaultConfig, - Node: config, - } - conf.Stack.DataDir = "/data" - conf.Stack.WSHost = "0.0.0.0" - conf.Stack.WSOrigins = []string{"*"} - conf.Stack.WSExposeAll = true - conf.Stack.P2P.EnableMsgEvents = false - conf.Stack.P2P.NoDiscovery = true - conf.Stack.P2P.NAT = nil - conf.Stack.NoUSB = true - - // listen on all interfaces on a given port, which we set when we - // initialise NodeConfig (usually a random port) - conf.Stack.P2P.ListenAddr = fmt.Sprintf(":%d", config.Port) - - node := &DockerNode{ - ExecNode: ExecNode{ - ID: config.ID, - Config: conf, - adapter: &d.ExecAdapter, - }, - } - node.newCmd = node.dockerCommand - d.ExecAdapter.nodes[node.ID] = &node.ExecNode - return node, nil -} - -// DockerNode wraps an ExecNode but exec's the current binary in a docker -// container rather than locally -type DockerNode struct { - ExecNode -} - -// dockerCommand returns a command which exec's the binary in a Docker -// container. -// -// It uses a shell so that we can pass the _P2P_NODE_CONFIG environment -// variable to the container using the --env flag. -func (n *DockerNode) dockerCommand() *exec.Cmd { - return exec.Command( - "sh", "-c", - fmt.Sprintf( - `exec docker run --interactive --env _P2P_NODE_CONFIG="${_P2P_NODE_CONFIG}" %s p2p-node %s %s`, - dockerImage, strings.Join(n.Config.Node.Services, ","), n.ID.String(), - ), - ) -} - -// dockerImage is the name of the Docker image which gets built to run the -// simulation node -const dockerImage = "p2p-node" - -// buildDockerImage builds the Docker image which is used to run the simulation -// node in a Docker container. -// -// It adds the current binary as "p2p-node" so that it runs execP2PNode -// when executed. -func buildDockerImage() error { - // create a directory to use as the build context - dir, err := ioutil.TempDir("", "p2p-docker") - if err != nil { - return err - } - defer os.RemoveAll(dir) - - // copy the current binary into the build context - bin, err := os.Open(reexec.Self()) - if err != nil { - return err - } - defer bin.Close() - dst, err := os.OpenFile(filepath.Join(dir, "self.bin"), os.O_WRONLY|os.O_CREATE, 0755) - if err != nil { - return err - } - defer dst.Close() - if _, err := io.Copy(dst, bin); err != nil { - return err - } - - // create the Dockerfile - dockerfile := []byte(` -FROM ubuntu:16.04 -RUN mkdir /data -ADD self.bin /bin/p2p-node - `) - if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dockerfile, 0644); err != nil { - return err - } - - // run 'docker build' - cmd := exec.Command("docker", "build", "-t", dockerImage, dir) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("error building docker image: %s", err) - } - - return nil -} diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index dc7d277cac..abb1967171 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -17,7 +17,7 @@ package adapters import ( - "bufio" + "bytes" "context" "crypto/ecdsa" "encoding/json" @@ -25,6 +25,7 @@ import ( "fmt" "io" "net" + "net/http" "os" "os/exec" "os/signal" @@ -43,12 +44,14 @@ import ( "golang.org/x/net/websocket" ) -// ExecAdapter is a NodeAdapter which runs simulation nodes by executing the -// current binary as a child process. -// -// An init hook is used so that the child process executes the node services -// (rather than whataver the main() function would normally do), see the -// execP2PNode function for more information. +func init() { + // Register a reexec function to start a simulation node when the current binary is + // executed as "p2p-node" (rather than whataver the main() function would normally do). + reexec.Register("p2p-node", execP2PNode) +} + +// ExecAdapter is a NodeAdapter which runs simulation nodes by executing the current binary +// as a child process. type ExecAdapter struct { // BaseDir is the directory under which the data directories for each // simulation node are created. @@ -150,15 +153,13 @@ func (n *ExecNode) Client() (*rpc.Client, error) { } // Start exec's the node passing the ID and service as command line arguments -// and the node config encoded as JSON in the _P2P_NODE_CONFIG environment -// variable +// and the node config encoded as JSON in an environment variable. func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { if n.Cmd != nil { return errors.New("already started") } defer func() { if err != nil { - log.Error("node failed to start", "err", err) n.Stop() } }() @@ -175,59 +176,78 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { return fmt.Errorf("error generating node config: %s", err) } - // use a pipe for stderr so we can both copy the node's stderr to - // os.Stderr and read the WebSocket address from the logs - stderrR, stderrW := io.Pipe() - stderr := io.MultiWriter(os.Stderr, stderrW) + // start the one-shot server that waits for startup information + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + statusURL, statusC := n.waitForStartupJSON(ctx) // start the node cmd := n.newCmd() cmd.Stdout = os.Stdout - cmd.Stderr = stderr - cmd.Env = append(os.Environ(), fmt.Sprintf("_P2P_NODE_CONFIG=%s", confData)) + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), + envStatusURL+"="+statusURL, + envNodeConfig+"="+string(confData), + ) if err := cmd.Start(); err != nil { return fmt.Errorf("error starting node: %s", err) } n.Cmd = cmd // read the WebSocket address from the stderr logs - var wsAddr string - wsAddrC := make(chan string) - go func() { - s := bufio.NewScanner(stderrR) - for s.Scan() { - if strings.Contains(s.Text(), "WebSocket endpoint opened") { - wsAddrC <- wsAddrPattern.FindString(s.Text()) - } - } - }() - select { - case wsAddr = <-wsAddrC: - if wsAddr == "" { - return errors.New("failed to read WebSocket address from stderr") - } - case <-time.After(10 * time.Second): - return errors.New("timed out waiting for WebSocket address on stderr") + status := <-statusC + if status.Err != "" { + return errors.New(status.Err) } - - // create the RPC client and load the node info - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - client, err := rpc.DialWebsocket(ctx, wsAddr, "") + client, err := rpc.DialWebsocket(ctx, status.WSEndpoint, "http://localhost") if err != nil { - return fmt.Errorf("error dialing rpc websocket: %s", err) + return fmt.Errorf("can't connect to RPC server: %v", err) } - var info p2p.NodeInfo - if err := client.CallContext(ctx, &info, "admin_nodeInfo"); err != nil { - return fmt.Errorf("error getting node info: %s", err) - } - n.client = client - n.wsAddr = wsAddr - n.Info = &info + // node ready :) + n.client = client + n.wsAddr = status.WSEndpoint + n.Info = status.NodeInfo return nil } +// waitForStartupJSON runs a one-shot HTTP server to receive a startup report. +func (n *ExecNode) waitForStartupJSON(ctx context.Context) (string, chan nodeStartupJSON) { + var ( + ch = make(chan nodeStartupJSON, 1) + quitOnce sync.Once + srv http.Server + ) + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ch <- nodeStartupJSON{Err: err.Error()} + return "", ch + } + quit := func(status nodeStartupJSON) { + quitOnce.Do(func() { + l.Close() + ch <- status + }) + } + srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var status nodeStartupJSON + if err := json.NewDecoder(r.Body).Decode(&status); err != nil { + status.Err = fmt.Sprintf("can't decode startup report: %v", err) + } + quit(status) + }) + // Run the HTTP server, but don't wait forever and shut it down + // if the context is canceled. + go srv.Serve(l) + go func() { + <-ctx.Done() + quit(nodeStartupJSON{Err: "didn't get startup report"}) + }() + + url := "http://" + l.Addr().String() + return url, ch +} + // execCommand returns a command which runs the node locally by exec'ing // the current binary but setting argv[0] to "p2p-node" so that the child // runs execP2PNode @@ -318,12 +338,6 @@ func (n *ExecNode) Snapshots() (map[string][]byte, error) { return snapshots, n.client.Call(&snapshots, "simulation_snapshot") } -func init() { - // register a reexec function to start a devp2p node when the current - // binary is executed as "p2p-node" - reexec.Register("p2p-node", execP2PNode) -} - // execNodeConfig is used to serialize the node configuration so it can be // passed to the child process as a JSON encoded environment variable type execNodeConfig struct { @@ -333,55 +347,69 @@ type execNodeConfig struct { PeerAddrs map[string]string `json:"peer_addrs,omitempty"` } -// ExternalIP gets an external IP address so that Enode URL is usable -func ExternalIP() net.IP { - addrs, err := net.InterfaceAddrs() - if err != nil { - log.Crit("error getting IP address", "err", err) - } - for _, addr := range addrs { - if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && !ip.IP.IsLinkLocalUnicast() { - return ip.IP - } - } - log.Warn("unable to determine explicit IP address, falling back to loopback") - return net.IP{127, 0, 0, 1} -} - -// execP2PNode starts a devp2p node when the current binary is executed with +// execP2PNode starts a simulation node when the current binary is executed with // argv[0] being "p2p-node", reading the service / ID from argv[1] / argv[2] -// and the node config from the _P2P_NODE_CONFIG environment variable +// and the node config from an environment variable. func execP2PNode() { glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat())) glogger.Verbosity(log.LvlInfo) log.Root().SetHandler(glogger) + statusURL := os.Getenv(envStatusURL) + if statusURL == "" { + log.Crit("missing " + envStatusURL) + } + // Start the node and gather startup report. + var status nodeStartupJSON + stack, stackErr := startExecNodeStack() + if stackErr != nil { + status.Err = stackErr.Error() + } else { + status.WSEndpoint = "ws://" + stack.WSEndpoint() + status.NodeInfo = stack.Server().NodeInfo() + } + + // Send status to the host. + statusJSON, _ := json.Marshal(status) + if _, err := http.Post(statusURL, "application/json", bytes.NewReader(statusJSON)); err != nil { + log.Crit("Can't post startup info", "url", statusURL, "err", err) + } + if stackErr != nil { + os.Exit(1) + } + + // Stop the stack if we get a SIGTERM signal. + go func() { + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGTERM) + defer signal.Stop(sigc) + <-sigc + log.Info("Received SIGTERM, shutting down...") + stack.Stop() + }() + stack.Wait() // Wait for the stack to exit. +} + +func startExecNodeStack() (*node.Node, error) { // read the services from argv serviceNames := strings.Split(os.Args[1], ",") // decode the config - confEnv := os.Getenv("_P2P_NODE_CONFIG") + confEnv := os.Getenv(envNodeConfig) if confEnv == "" { - log.Crit("missing _P2P_NODE_CONFIG") + return nil, fmt.Errorf("missing " + envNodeConfig) } var conf execNodeConfig if err := json.Unmarshal([]byte(confEnv), &conf); err != nil { - log.Crit("error decoding _P2P_NODE_CONFIG", "err", err) + return nil, fmt.Errorf("error decoding %s: %v", envNodeConfig, err) } conf.Stack.P2P.PrivateKey = conf.Node.PrivateKey conf.Stack.Logger = log.New("node.id", conf.Node.ID.String()) - if strings.HasPrefix(conf.Stack.P2P.ListenAddr, ":") { - conf.Stack.P2P.ListenAddr = ExternalIP().String() + conf.Stack.P2P.ListenAddr - } - if conf.Stack.WSHost == "0.0.0.0" { - conf.Stack.WSHost = ExternalIP().String() - } - // initialize the devp2p stack stack, err := node.New(&conf.Stack) if err != nil { - log.Crit("error creating node stack", "err", err) + return nil, fmt.Errorf("error creating node stack: %v", err) } // register the services, collecting them into a map so we can wrap @@ -390,7 +418,7 @@ func execP2PNode() { for _, name := range serviceNames { serviceFunc, exists := serviceFuncs[name] if !exists { - log.Crit("unknown node service", "name", name) + return nil, fmt.Errorf("unknown node service %q", err) } constructor := func(nodeCtx *node.ServiceContext) (node.Service, error) { ctx := &ServiceContext{ @@ -409,34 +437,35 @@ func execP2PNode() { return service, nil } if err := stack.Register(constructor); err != nil { - log.Crit("error starting service", "name", name, "err", err) + return stack, fmt.Errorf("error registering service %q: %v", name, err) } } // register the snapshot service - if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + err = stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { return &snapshotService{services}, nil - }); err != nil { - log.Crit("error starting snapshot service", "err", err) + }) + if err != nil { + return stack, fmt.Errorf("error starting snapshot service: %v", err) } // start the stack - if err := stack.Start(); err != nil { - log.Crit("error stating node stack", "err", err) + if err = stack.Start(); err != nil { + err = fmt.Errorf("error starting stack: %v", err) } + return stack, err +} - // stop the stack if we get a SIGTERM signal - go func() { - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGTERM) - defer signal.Stop(sigc) - <-sigc - log.Info("Received SIGTERM, shutting down...") - stack.Stop() - }() +const ( + envStatusURL = "_P2P_STATUS_URL" + envNodeConfig = "_P2P_NODE_CONFIG" +) - // wait for the stack to exit - stack.Wait() +// nodeStartupJSON is sent to the simulation host after startup. +type nodeStartupJSON struct { + Err string + WSEndpoint string + NodeInfo *p2p.NodeInfo } // snapshotService is a node.Service which wraps a list of services and diff --git a/p2p/simulations/adapters/ws.go b/p2p/simulations/adapters/ws.go deleted file mode 100644 index 979a21709e..0000000000 --- a/p2p/simulations/adapters/ws.go +++ /dev/null @@ -1,51 +0,0 @@ -package adapters - -import ( - "bufio" - "errors" - "io" - "regexp" - "strings" - "time" -) - -// wsAddrPattern is a regex used to read the WebSocket address from the node's -// log -var wsAddrPattern = regexp.MustCompile(`ws://[\d.:]+`) - -func matchWSAddr(str string) (string, bool) { - if !strings.Contains(str, "WebSocket endpoint opened") { - return "", false - } - - return wsAddrPattern.FindString(str), true -} - -// findWSAddr scans through reader r, looking for the log entry with -// WebSocket address information. -func findWSAddr(r io.Reader, timeout time.Duration) (string, error) { - ch := make(chan string) - - go func() { - s := bufio.NewScanner(r) - for s.Scan() { - addr, ok := matchWSAddr(s.Text()) - if ok { - ch <- addr - } - } - close(ch) - }() - - var wsAddr string - select { - case wsAddr = <-ch: - if wsAddr == "" { - return "", errors.New("empty result") - } - case <-time.After(timeout): - return "", errors.New("timed out") - } - - return wsAddr, nil -} diff --git a/p2p/simulations/adapters/ws_test.go b/p2p/simulations/adapters/ws_test.go deleted file mode 100644 index 0bb9ed2b2b..0000000000 --- a/p2p/simulations/adapters/ws_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package adapters - -import ( - "bytes" - "testing" - "time" -) - -func TestFindWSAddr(t *testing.T) { - line := `t=2018-05-02T19:00:45+0200 lvl=info msg="WebSocket endpoint opened" node.id=26c65a606d1125a44695bc08573190d047152b6b9a776ccbbe593e90f91444d9c1ebdadac6a775ad9fdd0923468a1d698ed3a842c1fb89c1bc0f9d4801f8c39c url=ws://127.0.0.1:59975` - buf := bytes.NewBufferString(line) - got, err := findWSAddr(buf, 10*time.Second) - if err != nil { - t.Fatalf("Failed to find addr: %v", err) - } - expected := `ws://127.0.0.1:59975` - - if got != expected { - t.Fatalf("Expected to get '%s', but got '%s'", expected, got) - } -} diff --git a/p2p/simulations/events.go b/p2p/simulations/events.go index f17958c689..9b2a990e07 100644 --- a/p2p/simulations/events.go +++ b/p2p/simulations/events.go @@ -58,6 +58,9 @@ type Event struct { // Msg is set if the type is EventTypeMsg Msg *Msg `json:"msg,omitempty"` + + //Optionally provide data (currently for simulation frontends only) + Data interface{} `json:"data"` } // NewEvent creates a new event for the given object which should be either a diff --git a/p2p/simulations/examples/ping-pong.go b/p2p/simulations/examples/ping-pong.go index 597cc950c9..cde2f3a677 100644 --- a/p2p/simulations/examples/ping-pong.go +++ b/p2p/simulations/examples/ping-pong.go @@ -70,14 +70,6 @@ func main() { log.Info("using exec adapter", "tmpdir", tmpdir) adapter = adapters.NewExecAdapter(tmpdir) - case "docker": - log.Info("using docker adapter") - var err error - adapter, err = adapters.NewDockerAdapter() - if err != nil { - log.Crit("error creating docker adapter", "err", err) - } - default: log.Crit(fmt.Sprintf("unknown node adapter %q", *adapterType)) } diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 7bdbad1b53..d9513caaa0 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -18,6 +18,7 @@ package simulations import ( "context" + "flag" "fmt" "math/rand" "net/http/httptest" @@ -28,13 +29,26 @@ import ( "time" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" + colorable "github.com/mattn/go-colorable" ) +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + // testService implements the node.Service interface and provides protocols // and APIs which are useful for testing nodes in a simulation network type testService struct { @@ -584,9 +598,26 @@ func TestHTTPNodeRPC(t *testing.T) { // TestHTTPSnapshot tests creating and loading network snapshots func TestHTTPSnapshot(t *testing.T) { // start the server - _, s := testHTTPServer(t) + network, s := testHTTPServer(t) defer s.Close() + var eventsDone = make(chan struct{}) + count := 1 + eventsDoneChan := make(chan *Event) + eventSub := network.Events().Subscribe(eventsDoneChan) + go func() { + defer eventSub.Unsubscribe() + for event := range eventsDoneChan { + if event.Type == EventTypeConn && !event.Control { + count-- + if count == 0 { + eventsDone <- struct{}{} + return + } + } + } + }() + // create a two-node network client := NewClient(s.URL) nodeCount := 2 @@ -620,7 +651,7 @@ func TestHTTPSnapshot(t *testing.T) { } states[i] = state } - + <-eventsDone // create a snapshot snap, err := client.CreateSnapshot() if err != nil { @@ -634,9 +665,23 @@ func TestHTTPSnapshot(t *testing.T) { } // create another network - _, s = testHTTPServer(t) + network2, s := testHTTPServer(t) defer s.Close() client = NewClient(s.URL) + count = 1 + eventSub = network2.Events().Subscribe(eventsDoneChan) + go func() { + defer eventSub.Unsubscribe() + for event := range eventsDoneChan { + if event.Type == EventTypeConn && !event.Control { + count-- + if count == 0 { + eventsDone <- struct{}{} + return + } + } + } + }() // subscribe to events so we can check them later events := make(chan *Event, 100) @@ -651,6 +696,7 @@ func TestHTTPSnapshot(t *testing.T) { if err := client.LoadSnapshot(snap); err != nil { t.Fatalf("error loading snapshot: %s", err) } + <-eventsDone // check the nodes and connection exists net, err := client.GetNetwork() @@ -676,6 +722,9 @@ func TestHTTPSnapshot(t *testing.T) { if conn.Other.String() != nodes[1].ID { t.Fatalf("expected connection to have other=%q, got other=%q", nodes[1].ID, conn.Other) } + if !conn.Up { + t.Fatal("should be up") + } // check the node states were restored for i, node := range nodes { diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 101ac09f85..92ccfde817 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -116,7 +116,7 @@ func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) Node: adapterNode, Config: conf, } - log.Trace(fmt.Sprintf("node %v created", conf.ID)) + log.Trace("Node created", "id", conf.ID) net.nodeMap[conf.ID] = len(net.Nodes) net.Nodes = append(net.Nodes, node) @@ -167,6 +167,7 @@ func (net *Network) Start(id enode.ID) error { func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) error { net.lock.Lock() defer net.lock.Unlock() + node := net.getNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) @@ -174,13 +175,13 @@ func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) if node.Up { return fmt.Errorf("node %v already up", id) } - log.Trace(fmt.Sprintf("starting node %v: %v using %v", id, node.Up, net.nodeAdapter.Name())) + log.Trace("Starting node", "id", id, "adapter", net.nodeAdapter.Name()) if err := node.Start(snapshots); err != nil { - log.Warn(fmt.Sprintf("start up failed: %v", err)) + log.Warn("Node startup failed", "id", id, "err", err) return err } node.Up = true - log.Info(fmt.Sprintf("started node %v: %v", id, node.Up)) + log.Info("Started node", "id", id) net.events.Send(NewEvent(node)) @@ -209,7 +210,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub defer net.lock.Unlock() node := net.getNode(id) if node == nil { - log.Error("Can not find node for id", "id", id) return } node.Up = false @@ -240,7 +240,7 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub case err := <-sub.Err(): if err != nil { - log.Error(fmt.Sprintf("error getting peer events for node %v", id), "err", err) + log.Error("Error in peer event subscription", "id", id, "err", err) } return } @@ -250,7 +250,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub // Stop stops the node with the given ID func (net *Network) Stop(id enode.ID) error { net.lock.Lock() - defer net.lock.Unlock() node := net.getNode(id) if node == nil { return fmt.Errorf("node %v does not exist", id) @@ -258,12 +257,17 @@ func (net *Network) Stop(id enode.ID) error { if !node.Up { return fmt.Errorf("node %v already down", id) } - if err := node.Stop(); err != nil { + node.Up = false + net.lock.Unlock() + + err := node.Stop() + if err != nil { + net.lock.Lock() + node.Up = true + net.lock.Unlock() return err } - node.Up = false - log.Info(fmt.Sprintf("stop node %v: %v", id, node.Up)) - + log.Info("Stopped node", "id", id, "err", err) net.events.Send(ControlEvent(node)) return nil } @@ -271,7 +275,7 @@ func (net *Network) Stop(id enode.ID) error { // Connect connects two nodes together by calling the "admin_addPeer" RPC // method on the "one" node so that it connects to the "other" node func (net *Network) Connect(oneID, otherID enode.ID) error { - log.Debug(fmt.Sprintf("connecting %s to %s", oneID, otherID)) + log.Debug("Connecting nodes with addPeer", "id", oneID, "other", otherID) conn, err := net.InitConn(oneID, otherID) if err != nil { return err @@ -481,10 +485,10 @@ func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) { err = conn.nodesUp() if err != nil { - log.Trace(fmt.Sprintf("nodes not up: %v", err)) + log.Trace("Nodes not up", "err", err) return nil, fmt.Errorf("nodes not up: %v", err) } - log.Debug("InitConn - connection initiated") + log.Debug("Connection initiated", "id", oneID, "other", otherID) conn.initiated = time.Now() return conn, nil } @@ -492,9 +496,9 @@ func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) { // Shutdown stops all nodes in the network and closes the quit channel func (net *Network) Shutdown() { for _, node := range net.Nodes { - log.Debug(fmt.Sprintf("stopping node %s", node.ID().TerminalString())) + log.Debug("Stopping node", "id", node.ID()) if err := node.Stop(); err != nil { - log.Warn(fmt.Sprintf("error stopping node %s", node.ID().TerminalString()), "err", err) + log.Warn("Can't stop node", "id", node.ID(), "err", err) } } close(net.quitc) @@ -640,11 +644,18 @@ type NodeSnapshot struct { // Snapshot creates a network snapshot func (net *Network) Snapshot() (*Snapshot, error) { + return net.snapshot(nil, nil) +} + +func (net *Network) SnapshotWithServices(addServices []string, removeServices []string) (*Snapshot, error) { + return net.snapshot(addServices, removeServices) +} + +func (net *Network) snapshot(addServices []string, removeServices []string) (*Snapshot, error) { net.lock.Lock() defer net.lock.Unlock() snap := &Snapshot{ Nodes: make([]NodeSnapshot, len(net.Nodes)), - Conns: make([]Conn, len(net.Conns)), } for i, node := range net.Nodes { snap.Nodes[i] = NodeSnapshot{Node: *node} @@ -656,9 +667,40 @@ func (net *Network) Snapshot() (*Snapshot, error) { return nil, err } snap.Nodes[i].Snapshots = snapshots + for _, addSvc := range addServices { + haveSvc := false + for _, svc := range snap.Nodes[i].Node.Config.Services { + if svc == addSvc { + haveSvc = true + break + } + } + if !haveSvc { + snap.Nodes[i].Node.Config.Services = append(snap.Nodes[i].Node.Config.Services, addSvc) + } + } + if len(removeServices) > 0 { + var cleanedServices []string + for _, svc := range snap.Nodes[i].Node.Config.Services { + haveSvc := false + for _, rmSvc := range removeServices { + if rmSvc == svc { + haveSvc = true + break + } + } + if !haveSvc { + cleanedServices = append(cleanedServices, svc) + } + + } + snap.Nodes[i].Node.Config.Services = cleanedServices + } } - for i, conn := range net.Conns { - snap.Conns[i] = *conn + for _, conn := range net.Conns { + if conn.Up { + snap.Conns = append(snap.Conns, *conn) + } } return snap, nil } @@ -708,18 +750,18 @@ func (net *Network) Subscribe(events chan *Event) { } func (net *Network) executeControlEvent(event *Event) { - log.Trace("execute control event", "type", event.Type, "event", event) + log.Trace("Executing control event", "type", event.Type, "event", event) switch event.Type { case EventTypeNode: if err := net.executeNodeEvent(event); err != nil { - log.Error("error executing node event", "event", event, "err", err) + log.Error("Error executing node event", "event", event, "err", err) } case EventTypeConn: if err := net.executeConnEvent(event); err != nil { - log.Error("error executing conn event", "event", event, "err", err) + log.Error("Error executing conn event", "event", event, "err", err) } case EventTypeMsg: - log.Warn("ignoring control msg event") + log.Warn("Ignoring control msg event") } } diff --git a/params/config.go b/params/config.go index a9e631cde4..007e4a66d8 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 193, - SectionHead: common.HexToHash("0xc2d574295ecedc4d58530ae24c31a5a98be7d2b3327fba0dd0f4ed3913828a55"), - CHTRoot: common.HexToHash("0x5d1027dfae688c77376e842679ceada87fd94738feb9b32ef165473bfbbb317b"), - BloomRoot: common.HexToHash("0xd38be1a06aabd568e10957fee4fcc523bc64996bcf31bae3f55f86e0a583919f"), + SectionIndex: 203, + SectionHead: common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"), + CHTRoot: common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"), + BloomRoot: common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -66,17 +66,17 @@ var ( EIP155Block: big.NewInt(10), EIP158Block: big.NewInt(10), ByzantiumBlock: big.NewInt(1700000), - ConstantinopleBlock: nil, + ConstantinopleBlock: big.NewInt(4230000), Ethash: new(EthashConfig), } // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 123, - SectionHead: common.HexToHash("0xa372a53decb68ce453da12bea1c8ee7b568b276aa2aab94d9060aa7c81fc3dee"), - CHTRoot: common.HexToHash("0x6b02e7fada79cd2a80d4b3623df9c44384d6647fc127462e1c188ccd09ece87b"), - BloomRoot: common.HexToHash("0xf2d27490914968279d6377d42868928632573e823b5d1d4a944cba6009e16259"), + SectionIndex: 134, + SectionHead: common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"), + CHTRoot: common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"), + BloomRoot: common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 91, - SectionHead: common.HexToHash("0x435b7b2d8a7922f3b9a522f2fb02730e95e0e1782f0f5443894d5415bba37154"), - CHTRoot: common.HexToHash("0x0664bf7ecccfb6775c4eca6f0f264fb5282a22754a2135a1ac4bff2ef02898dd"), - BloomRoot: common.HexToHash("0x2a64df2400c3a2cb6400639bb6ed29389abdb4d93e2e525aa7c21f38767cd96f"), + SectionIndex: 100, + SectionHead: common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"), + CHTRoot: common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"), + BloomRoot: common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced diff --git a/params/version.go b/params/version.go index e87d7dc35c..b9dcc2a842 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 17 // Patch version component of the current release + VersionPatch = 19 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rpc/client.go b/rpc/client.go index d96189a2d8..6254c95ffd 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -25,7 +25,6 @@ import ( "fmt" "net" "net/url" - "os" "reflect" "strconv" "strings" @@ -118,7 +117,8 @@ type Client struct { // for dispatch close chan struct{} - didQuit chan struct{} // closed when client quits + closing chan struct{} // closed when client is quitting + didClose chan struct{} // closed when client quits reconnected chan net.Conn // where write/reconnect sends the new connection readErr chan error // errors from read readResp chan []*jsonrpcMessage // valid messages from read @@ -181,45 +181,6 @@ func DialContext(ctx context.Context, rawurl string) (*Client, error) { } } -type StdIOConn struct{} - -func (io StdIOConn) Read(b []byte) (n int, err error) { - return os.Stdin.Read(b) -} - -func (io StdIOConn) Write(b []byte) (n int, err error) { - return os.Stdout.Write(b) -} - -func (io StdIOConn) Close() error { - return nil -} - -func (io StdIOConn) LocalAddr() net.Addr { - return &net.UnixAddr{Name: "stdio", Net: "stdio"} -} - -func (io StdIOConn) RemoteAddr() net.Addr { - return &net.UnixAddr{Name: "stdio", Net: "stdio"} -} - -func (io StdIOConn) SetDeadline(t time.Time) error { - return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} -} - -func (io StdIOConn) SetReadDeadline(t time.Time) error { - return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} -} - -func (io StdIOConn) SetWriteDeadline(t time.Time) error { - return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} -} -func DialStdIO(ctx context.Context) (*Client, error) { - return newClient(ctx, func(_ context.Context) (net.Conn, error) { - return StdIOConn{}, nil - }) -} - func newClient(initctx context.Context, connectFunc func(context.Context) (net.Conn, error)) (*Client, error) { conn, err := connectFunc(initctx) if err != nil { @@ -231,7 +192,8 @@ func newClient(initctx context.Context, connectFunc func(context.Context) (net.C isHTTP: isHTTP, connectFunc: connectFunc, close: make(chan struct{}), - didQuit: make(chan struct{}), + closing: make(chan struct{}), + didClose: make(chan struct{}), reconnected: make(chan net.Conn), readErr: make(chan error), readResp: make(chan []*jsonrpcMessage), @@ -268,8 +230,8 @@ func (c *Client) Close() { } select { case c.close <- struct{}{}: - <-c.didQuit - case <-c.didQuit: + <-c.didClose + case <-c.didClose: } } @@ -469,7 +431,9 @@ func (c *Client) send(ctx context.Context, op *requestOp, msg interface{}) error // This can happen if the client is overloaded or unable to keep up with // subscription notifications. return ctx.Err() - case <-c.didQuit: + case <-c.closing: + return ErrClientQuit + case <-c.didClose: return ErrClientQuit } } @@ -504,7 +468,7 @@ func (c *Client) reconnect(ctx context.Context) error { case c.reconnected <- newconn: c.writeConn = newconn return nil - case <-c.didQuit: + case <-c.didClose: newconn.Close() return ErrClientQuit } @@ -522,8 +486,9 @@ func (c *Client) dispatch(conn net.Conn) { requestOpLock = c.requestOp // nil while the send lock is held reading = true // if true, a read loop is running ) - defer close(c.didQuit) + defer close(c.didClose) defer func() { + close(c.closing) c.closeRequestOps(ErrClientQuit) conn.Close() if reading { diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go index 9c21c12d5d..3bb8717b80 100644 --- a/rpc/client_example_test.go +++ b/rpc/client_example_test.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// In this example, our client whishes to track the latest 'block number' +// In this example, our client wishes to track the latest 'block number' // known to the server. The server supports two methods: // // eth_getBlockByNumber("latest", {}) diff --git a/rpc/client_test.go b/rpc/client_test.go index 4f354d389e..a8195c0af8 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -323,6 +323,30 @@ func TestClientSubscribeClose(t *testing.T) { } } +// This test reproduces https://github.com/ethereum/go-ethereum/issues/17837 where the +// client hangs during shutdown when Unsubscribe races with Client.Close. +func TestClientCloseUnsubscribeRace(t *testing.T) { + service := &NotificationTestService{} + server := newTestServer("eth", service) + defer server.Stop() + + for i := 0; i < 20; i++ { + client := DialInProc(server) + nc := make(chan int) + sub, err := client.EthSubscribe(context.Background(), nc, "someSubscription", 3, 1) + if err != nil { + t.Fatal(err) + } + go client.Close() + go sub.Unsubscribe() + select { + case <-sub.Err(): + case <-time.After(5 * time.Second): + t.Fatal("subscription not closed within timeout") + } + } +} + // This test checks that Client doesn't lock up when a single subscriber // doesn't read subscription events. func TestClientNotificationStorm(t *testing.T) { diff --git a/rpc/doc.go b/rpc/doc.go index 9a6c4abbce..c60381b5ae 100644 --- a/rpc/doc.go +++ b/rpc/doc.go @@ -32,7 +32,7 @@ An example method: func (s *CalcService) Add(a, b int) (int, error) When the returned error isn't nil the returned integer is ignored and the error is -send back to the client. Otherwise the returned integer is send back to the client. +sent back to the client. Otherwise the returned integer is sent back to the client. Optional arguments are supported by accepting pointer values as arguments. E.g. if we want to do the addition in an optional finite field we can accept a mod diff --git a/rpc/ipc_js.go b/rpc/ipc_js.go new file mode 100644 index 0000000000..eceef050e7 --- /dev/null +++ b/rpc/ipc_js.go @@ -0,0 +1,37 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build js + +package rpc + +import ( + "context" + "errors" + "net" +) + +var errNotSupported = errors.New("rpc: not supported") + +// ipcListen will create a named pipe on the given endpoint. +func ipcListen(endpoint string) (net.Listener, error) { + return nil, errNotSupported +} + +// newIPCConnection will connect to a named pipe with the given endpoint as name. +func newIPCConnection(ctx context.Context, endpoint string) (net.Conn, error) { + return nil, errNotSupported +} diff --git a/rpc/stdio.go b/rpc/stdio.go new file mode 100644 index 0000000000..ea552cca28 --- /dev/null +++ b/rpc/stdio.go @@ -0,0 +1,66 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rpc + +import ( + "context" + "errors" + "net" + "os" + "time" +) + +// DialStdIO creates a client on stdin/stdout. +func DialStdIO(ctx context.Context) (*Client, error) { + return newClient(ctx, func(_ context.Context) (net.Conn, error) { + return stdioConn{}, nil + }) +} + +type stdioConn struct{} + +func (io stdioConn) Read(b []byte) (n int, err error) { + return os.Stdin.Read(b) +} + +func (io stdioConn) Write(b []byte) (n int, err error) { + return os.Stdout.Write(b) +} + +func (io stdioConn) Close() error { + return nil +} + +func (io stdioConn) LocalAddr() net.Addr { + return &net.UnixAddr{Name: "stdio", Net: "stdio"} +} + +func (io stdioConn) RemoteAddr() net.Addr { + return &net.UnixAddr{Name: "stdio", Net: "stdio"} +} + +func (io stdioConn) SetDeadline(t time.Time) error { + return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} +} + +func (io stdioConn) SetReadDeadline(t time.Time) error { + return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} +} + +func (io stdioConn) SetWriteDeadline(t time.Time) error { + return &net.OpError{Op: "set", Net: "stdio", Source: nil, Addr: nil, Err: errors.New("deadline not supported")} +} diff --git a/rpc/subscription.go b/rpc/subscription.go index 6ce7befa1d..6bbb6f75d2 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -52,9 +52,10 @@ type notifierKey struct{} // Server callbacks use the notifier to send notifications. type Notifier struct { codec ServerCodec - subMu sync.RWMutex // guards active and inactive maps + subMu sync.Mutex active map[ID]*Subscription inactive map[ID]*Subscription + buffer map[ID][]interface{} // unsent notifications of inactive subscriptions } // newNotifier creates a new notifier that can be used to send subscription @@ -64,6 +65,7 @@ func newNotifier(codec ServerCodec) *Notifier { codec: codec, active: make(map[ID]*Subscription), inactive: make(map[ID]*Subscription), + buffer: make(map[ID][]interface{}), } } @@ -88,20 +90,26 @@ func (n *Notifier) CreateSubscription() *Subscription { // Notify sends a notification to the client with the given data as payload. // If an error occurs the RPC connection is closed and the error is returned. func (n *Notifier) Notify(id ID, data interface{}) error { - n.subMu.RLock() - defer n.subMu.RUnlock() + n.subMu.Lock() + defer n.subMu.Unlock() - sub, active := n.active[id] - if active { - notification := n.codec.CreateNotification(string(id), sub.namespace, data) - if err := n.codec.Write(notification); err != nil { - n.codec.Close() - return err - } + if sub, active := n.active[id]; active { + n.send(sub, data) + } else { + n.buffer[id] = append(n.buffer[id], data) } return nil } +func (n *Notifier) send(sub *Subscription, data interface{}) error { + notification := n.codec.CreateNotification(string(sub.ID), sub.namespace, data) + err := n.codec.Write(notification) + if err != nil { + n.codec.Close() + } + return err +} + // Closed returns a channel that is closed when the RPC connection is closed. func (n *Notifier) Closed() <-chan interface{} { return n.codec.Closed() @@ -127,9 +135,15 @@ func (n *Notifier) unsubscribe(id ID) error { func (n *Notifier) activate(id ID, namespace string) { n.subMu.Lock() defer n.subMu.Unlock() + if sub, found := n.inactive[id]; found { sub.namespace = namespace n.active[id] = sub delete(n.inactive, id) + // Send buffered notifications. + for _, data := range n.buffer[id] { + n.send(sub, data) + } + delete(n.buffer, id) } } diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index 0ba177e63b..24febc9190 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -27,9 +27,8 @@ import ( ) type NotificationTestService struct { - mu sync.Mutex - unsubscribed bool - + mu sync.Mutex + unsubscribed chan string gotHangSubscriptionReq chan struct{} unblockHangSubscription chan struct{} } @@ -38,16 +37,10 @@ func (s *NotificationTestService) Echo(i int) int { return i } -func (s *NotificationTestService) wasUnsubCallbackCalled() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.unsubscribed -} - func (s *NotificationTestService) Unsubscribe(subid string) { - s.mu.Lock() - s.unsubscribed = true - s.mu.Unlock() + if s.unsubscribed != nil { + s.unsubscribed <- subid + } } func (s *NotificationTestService) SomeSubscription(ctx context.Context, n, val int) (*Subscription, error) { @@ -65,7 +58,6 @@ func (s *NotificationTestService) SomeSubscription(ctx context.Context, n, val i // test expects n events, if we begin sending event immediately some events // will probably be dropped since the subscription ID might not be send to // the client. - time.Sleep(5 * time.Second) for i := 0; i < n; i++ { if err := notifier.Notify(subscription.ID, val+i); err != nil { return @@ -74,13 +66,10 @@ func (s *NotificationTestService) SomeSubscription(ctx context.Context, n, val i select { case <-notifier.Closed(): - s.mu.Lock() - s.unsubscribed = true - s.mu.Unlock() case <-subscription.Err(): - s.mu.Lock() - s.unsubscribed = true - s.mu.Unlock() + } + if s.unsubscribed != nil { + s.unsubscribed <- string(subscription.ID) } }() @@ -107,7 +96,7 @@ func (s *NotificationTestService) HangSubscription(ctx context.Context, val int) func TestNotifications(t *testing.T) { server := NewServer() - service := &NotificationTestService{} + service := &NotificationTestService{unsubscribed: make(chan string)} if err := server.RegisterName("eth", service); err != nil { t.Fatalf("unable to register test service %v", err) @@ -157,10 +146,10 @@ func TestNotifications(t *testing.T) { } clientConn.Close() // causes notification unsubscribe callback to be called - time.Sleep(1 * time.Second) - - if !service.wasUnsubCallbackCalled() { - t.Error("unsubscribe callback not called after closing connection") + select { + case <-service.unsubscribed: + case <-time.After(1 * time.Second): + t.Fatal("Unsubscribe not called after one second") } } @@ -227,18 +216,19 @@ func waitForMessages(t *testing.T, in *json.Decoder, successes chan<- jsonSucces // for multiple different namespaces. func TestSubscriptionMultipleNamespaces(t *testing.T) { var ( - namespaces = []string{"eth", "shh", "bzz"} + namespaces = []string{"eth", "shh", "bzz"} + service = NotificationTestService{} + subCount = len(namespaces) * 2 + notificationCount = 3 + server = NewServer() - service = NotificationTestService{} clientConn, serverConn = net.Pipe() - - out = json.NewEncoder(clientConn) - in = json.NewDecoder(clientConn) - successes = make(chan jsonSuccessResponse) - failures = make(chan jsonErrResponse) - notifications = make(chan jsonNotification) - - errors = make(chan error, 10) + out = json.NewEncoder(clientConn) + in = json.NewDecoder(clientConn) + successes = make(chan jsonSuccessResponse) + failures = make(chan jsonErrResponse) + notifications = make(chan jsonNotification) + errors = make(chan error, 10) ) // setup and start server @@ -255,13 +245,12 @@ func TestSubscriptionMultipleNamespaces(t *testing.T) { go waitForMessages(t, in, successes, failures, notifications, errors) // create subscriptions one by one - n := 3 for i, namespace := range namespaces { request := map[string]interface{}{ "id": i, "method": fmt.Sprintf("%s_subscribe", namespace), "version": "2.0", - "params": []interface{}{"someSubscription", n, i}, + "params": []interface{}{"someSubscription", notificationCount, i}, } if err := out.Encode(&request); err != nil { @@ -276,7 +265,7 @@ func TestSubscriptionMultipleNamespaces(t *testing.T) { "id": i, "method": fmt.Sprintf("%s_subscribe", namespace), "version": "2.0", - "params": []interface{}{"someSubscription", n, i}, + "params": []interface{}{"someSubscription", notificationCount, i}, }) } @@ -285,46 +274,40 @@ func TestSubscriptionMultipleNamespaces(t *testing.T) { } timeout := time.After(30 * time.Second) - subids := make(map[string]string, 2*len(namespaces)) - count := make(map[string]int, 2*len(namespaces)) - - for { - done := true - for id := range count { - if count, found := count[id]; !found || count < (2*n) { + subids := make(map[string]string, subCount) + count := make(map[string]int, subCount) + allReceived := func() bool { + done := len(count) == subCount + for _, c := range count { + if c < notificationCount { done = false } } + return done + } - if done && len(count) == len(namespaces) { - break - } - + for !allReceived() { select { - case err := <-errors: - t.Fatal(err) case suc := <-successes: // subscription created subids[namespaces[int(suc.Id.(float64))]] = suc.Result.(string) + case notification := <-notifications: + count[notification.Params.Subscription]++ + case err := <-errors: + t.Fatal(err) case failure := <-failures: t.Errorf("received error: %v", failure.Error) - case notification := <-notifications: - if cnt, found := count[notification.Params.Subscription]; found { - count[notification.Params.Subscription] = cnt + 1 - } else { - count[notification.Params.Subscription] = 1 - } case <-timeout: for _, namespace := range namespaces { subid, found := subids[namespace] if !found { - t.Errorf("Subscription for '%s' not created", namespace) + t.Errorf("subscription for %q not created", namespace) continue } - if count, found := count[subid]; !found || count < n { - t.Errorf("Didn't receive all notifications (%d<%d) in time for namespace '%s'", count, n, namespace) + if count, found := count[subid]; !found || count < notificationCount { + t.Errorf("didn't receive all notifications (%d<%d) in time for namespace %q", count, notificationCount, namespace) } } - return + t.Fatal("timed out") } } } diff --git a/signer/core/abihelper_test.go b/signer/core/abihelper_test.go index 2afeec73e6..878210be1f 100644 --- a/signer/core/abihelper_test.go +++ b/signer/core/abihelper_test.go @@ -38,13 +38,13 @@ func verify(t *testing.T, jsondata, calldata string, exp []interface{}) { cd := common.Hex2Bytes(calldata) sigdata, argdata := cd[:4], cd[4:] method, err := abispec.MethodById(sigdata) - if err != nil { t.Fatal(err) } - data, err := method.Inputs.UnpackValues(argdata) - + if err != nil { + t.Fatal(err) + } if len(data) != len(exp) { t.Fatalf("Mismatched length, expected %d, got %d", len(exp), len(data)) } diff --git a/signer/core/api.go b/signer/core/api.go index 1da86991ed..2b96cdb5f3 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -36,6 +36,9 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +// numberOfAccountsToDerive For hardware wallets, the number of accounts to derive +const numberOfAccountsToDerive = 10 + // ExternalAPI defines the external API through which signing requests are made. type ExternalAPI interface { // List available accounts @@ -79,6 +82,9 @@ type SignerUI interface { // OnSignerStartup is invoked when the signer boots, and tells the UI info about external API location and version // information OnSignerStartup(info StartupInfo) + // OnInputRequried is invoked when clef requires user input, for example master password or + // pin-code for unlocking hardware wallets + OnInputRequired(info UserInputRequest) (UserInputResponse, error) } // SignerAPI defines the actual implementation of ExternalAPI @@ -191,9 +197,23 @@ type ( Message struct { Text string `json:"text"` } + PasswordRequest struct { + Prompt string `json:"prompt"` + } + PasswordResponse struct { + Password string `json:"password"` + } StartupInfo struct { Info map[string]interface{} `json:"info"` } + UserInputRequest struct { + Prompt string `json:"prompt"` + Title string `json:"title"` + IsPassword bool `json:"isPassword"` + } + UserInputResponse struct { + Text string `json:"text"` + } ) var ErrRequestDenied = errors.New("Request denied") @@ -215,6 +235,9 @@ func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abi if len(ksLocation) > 0 { backends = append(backends, keystore.NewKeyStore(ksLocation, n, p)) } + if advancedMode { + log.Info("Clef is in advanced mode: will warn instead of reject") + } if !noUSB { // Start a USB hub for Ledger hardware wallets if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { @@ -231,10 +254,94 @@ func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abi log.Debug("Trezor support enabled") } } - if advancedMode { - log.Info("Clef is in advanced mode: will warn instead of reject") + signer := &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb), !advancedMode} + if !noUSB { + signer.startUSBListener() } - return &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb), !advancedMode} + return signer +} +func (api *SignerAPI) openTrezor(url accounts.URL) { + resp, err := api.UI.OnInputRequired(UserInputRequest{ + Prompt: "Pin required to open Trezor wallet\n" + + "Look at the device for number positions\n\n" + + "7 | 8 | 9\n" + + "--+---+--\n" + + "4 | 5 | 6\n" + + "--+---+--\n" + + "1 | 2 | 3\n\n", + IsPassword: true, + Title: "Trezor unlock", + }) + if err != nil { + log.Warn("failed getting trezor pin", "err", err) + return + } + // We're using the URL instead of the pointer to the + // Wallet -- perhaps it is not actually present anymore + w, err := api.am.Wallet(url.String()) + if err != nil { + log.Warn("wallet unavailable", "url", url) + return + } + err = w.Open(resp.Text) + if err != nil { + log.Warn("failed to open wallet", "wallet", url, "err", err) + return + } + +} + +// startUSBListener starts a listener for USB events, for hardware wallet interaction +func (api *SignerAPI) startUSBListener() { + events := make(chan accounts.WalletEvent, 16) + am := api.am + am.Subscribe(events) + go func() { + + // Open any wallets already attached + for _, wallet := range am.Wallets() { + if err := wallet.Open(""); err != nil { + log.Warn("Failed to open wallet", "url", wallet.URL(), "err", err) + if err == usbwallet.ErrTrezorPINNeeded { + go api.openTrezor(wallet.URL()) + } + } + } + // Listen for wallet event till termination + for event := range events { + switch event.Kind { + case accounts.WalletArrived: + if err := event.Wallet.Open(""); err != nil { + log.Warn("New wallet appeared, failed to open", "url", event.Wallet.URL(), "err", err) + if err == usbwallet.ErrTrezorPINNeeded { + go api.openTrezor(event.Wallet.URL()) + } + } + case accounts.WalletOpened: + status, _ := event.Wallet.Status() + log.Info("New wallet appeared", "url", event.Wallet.URL(), "status", status) + + derivationPath := accounts.DefaultBaseDerivationPath + if event.Wallet.URL().Scheme == "ledger" { + derivationPath = accounts.DefaultLedgerBaseDerivationPath + } + var nextPath = derivationPath + // Derive first N accounts, hardcoded for now + for i := 0; i < numberOfAccountsToDerive; i++ { + acc, err := event.Wallet.Derive(nextPath, true) + if err != nil { + log.Warn("account derivation failed", "error", err) + } else { + log.Info("derived account", "address", acc.Address) + } + nextPath[len(nextPath)-1]++ + } + case accounts.WalletDropped: + log.Info("Old wallet dropped", "url", event.Wallet.URL()) + event.Wallet.Close() + } + } + }() } // List returns the set of wallet this signer manages. Each wallet can contain diff --git a/signer/core/api_test.go b/signer/core/api_test.go index 70aa9aa94e..a8aa23896e 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -19,6 +19,7 @@ package core import ( "bytes" "context" + "errors" "fmt" "io/ioutil" "math/big" @@ -41,6 +42,10 @@ type HeadlessUI struct { controller chan string } +func (ui *HeadlessUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { + return UserInputResponse{}, errors.New("not implemented") +} + func (ui *HeadlessUI) OnSignerStartup(info StartupInfo) { } diff --git a/signer/core/cliui.go b/signer/core/cliui.go index cc237612eb..940f1f43aa 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -83,6 +83,22 @@ func (ui *CommandlineUI) readPasswordText(inputstring string) string { return string(text) } +func (ui *CommandlineUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { + fmt.Println(info.Title) + fmt.Println(info.Prompt) + if info.IsPassword { + text, err := terminal.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + log.Error("Failed to read password", "err", err) + } + fmt.Println("-----------------------") + return UserInputResponse{string(text)}, err + } + text := ui.readString() + fmt.Println("-----------------------") + return UserInputResponse{text}, nil +} + // confirm returns true if user enters 'Yes', otherwise false func (ui *CommandlineUI) confirm() bool { fmt.Printf("Approve? [y/N]:\n") diff --git a/signer/core/stdioui.go b/signer/core/stdioui.go index 5640ed03bd..64032386fc 100644 --- a/signer/core/stdioui.go +++ b/signer/core/stdioui.go @@ -111,3 +111,11 @@ func (ui *StdIOUI) OnSignerStartup(info StartupInfo) { log.Info("Error calling 'OnSignerStartup'", "exc", err.Error(), "info", info) } } +func (ui *StdIOUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { + var result UserInputResponse + err := ui.dispatch("OnInputRequired", info, &result) + if err != nil { + log.Info("Error calling 'OnInputRequired'", "exc", err.Error(), "info", info) + } + return result, err +} diff --git a/signer/rules/rules.go b/signer/rules/rules.go index 711e2dddec..07c34db220 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -194,6 +194,11 @@ func (r *rulesetUI) ApproveImport(request *core.ImportRequest) (core.ImportRespo return r.next.ApproveImport(request) } +// OnInputRequired not handled by rules +func (r *rulesetUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) { + return r.next.OnInputRequired(info) +} + func (r *rulesetUI) ApproveListing(request *core.ListRequest) (core.ListResponse, error) { jsonreq, err := json.Marshal(request) approved, err := r.checkApproval("ApproveListing", jsonreq, err) @@ -222,6 +227,7 @@ func (r *rulesetUI) ShowInfo(message string) { log.Info(message) r.next.ShowInfo(message) } + func (r *rulesetUI) OnSignerStartup(info core.StartupInfo) { jsonInfo, err := json.Marshal(info) if err != nil { diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index b6060eba73..0b520a15bf 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -74,9 +74,17 @@ func mixAddr(a string) (*common.MixedcaseAddress, error) { type alwaysDenyUI struct{} +func (alwaysDenyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) { + return core.UserInputResponse{}, nil +} + func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) { } +func (alwaysDenyUI) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) { + return core.PasswordResponse{}, nil +} + func (alwaysDenyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { return core.SignTxResponse{Transaction: request.Transaction, Approved: false, Password: ""}, nil } @@ -143,7 +151,7 @@ func TestListRequest(t *testing.T) { t.Errorf("Couldn't create evaluator %v", err) return } - resp, err := r.ApproveListing(&core.ListRequest{ + resp, _ := r.ApproveListing(&core.ListRequest{ Accounts: accs, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, }) @@ -200,6 +208,11 @@ type dummyUI struct { calls []string } +func (d *dummyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) { + d.calls = append(d.calls, "OnInputRequired") + return core.UserInputResponse{}, nil +} + func (d *dummyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { d.calls = append(d.calls, "ApproveTx") return core.SignTxResponse{}, core.ErrRequestDenied @@ -241,6 +254,11 @@ func (d *dummyUI) ShowInfo(message string) { func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { d.calls = append(d.calls, "OnApprovedTx") } + +func (d *dummyUI) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) { + return core.PasswordResponse{}, nil +} + func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { } @@ -497,7 +515,7 @@ func TestLimitWindow(t *testing.T) { r.OnApprovedTx(response) } // Fourth should fail - resp, err := r.ApproveTx(dummyTx(h)) + resp, _ := r.ApproveTx(dummyTx(h)) if resp.Approved { t.Errorf("Expected check to resolve to 'Reject'") } @@ -509,9 +527,18 @@ type dontCallMe struct { t *testing.T } +func (d *dontCallMe) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) { + d.t.Fatalf("Did not expect next-handler to be called") + return core.UserInputResponse{}, nil +} + func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) { } +func (d *dontCallMe) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) { + return core.PasswordResponse{}, nil +} + func (d *dontCallMe) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { d.t.Fatalf("Did not expect next-handler to be called") return core.SignTxResponse{}, core.ErrRequestDenied @@ -582,8 +609,8 @@ func TestContextIsCleared(t *testing.T) { t.Fatalf("Failed to load bootstrap js: %v", err) } tx := dummyTxWithV(0) - r1, err := r.ApproveTx(tx) - r2, err := r.ApproveTx(tx) + r1, _ := r.ApproveTx(tx) + r2, _ := r.ApproveTx(tx) if r1.Approved != r2.Approved { t.Errorf("Expected execution context to be cleared between executions") } diff --git a/swarm/OWNERS b/swarm/OWNERS index 774cd7db9d..d4204e08cf 100644 --- a/swarm/OWNERS +++ b/swarm/OWNERS @@ -22,5 +22,5 @@ swarm ├── storage ─────────────── ethersphere │ ├── encryption ──────── @gbalint, @zelig, @nagydani │ ├── mock ────────────── @janos -│ └── mru ─────────────── @nolash +│ └── feed ────────────── @nolash, @jpeletier └── testutil ────────────── @lmars \ No newline at end of file diff --git a/swarm/api/act.go b/swarm/api/act.go index 52d9098271..e54369f9a8 100644 --- a/swarm/api/act.go +++ b/swarm/api/act.go @@ -458,6 +458,9 @@ func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees return nil, nil, nil, err } sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt) + if err != nil { + return nil, nil, nil, err + } hasher := sha3.NewKeccak256() hasher.Write(append(sessionKey, 0)) diff --git a/swarm/api/api.go b/swarm/api/api.go index d7b6d84190..7bb631967c 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -16,6 +16,9 @@ package api +//go:generate mimegen --types=./../../cmd/swarm/mimegen/mime.types --package=api --out=gen_mime.go +//go:generate gofmt -s -w gen_mime.go + import ( "archive/tar" "context" @@ -42,7 +45,9 @@ import ( "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + opentracing "github.com/opentracing/opentracing-go" ) @@ -231,18 +236,18 @@ on top of the FileStore it is the public interface of the FileStore which is included in the ethereum stack */ type API struct { - resource *mru.Handler + feed *feed.Handler fileStore *storage.FileStore dns Resolver Decryptor func(context.Context, string) DecryptFunc } // NewAPI the api constructor initialises a new API instance. -func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler, pk *ecdsa.PrivateKey) (self *API) { +func NewAPI(fileStore *storage.FileStore, dns Resolver, feedHandler *feed.Handler, pk *ecdsa.PrivateKey) (self *API) { self = &API{ fileStore: fileStore, dns: dns, - resource: resourceHandler, + feed: feedHandler, Decryptor: func(ctx context.Context, credentials string) DecryptFunc { return self.doDecrypt(ctx, credentials, pk) }, @@ -399,83 +404,60 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage return a.Get(ctx, decrypt, adr, entry.Path) } - // we need to do some extra work if this is a mutable resource manifest - if entry.ContentType == ResourceContentType { - - // get the resource rootAddr - log.Trace("resource type", "menifestAddr", manifestAddr, "hash", entry.Hash) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - rootAddr := storage.Address(common.FromHex(entry.Hash)) - rsrc, err := a.resource.Load(ctx, rootAddr) + // we need to do some extra work if this is a Swarm feed manifest + if entry.ContentType == FeedContentType { + if entry.Feed == nil { + return reader, mimeType, status, nil, fmt.Errorf("Cannot decode Feed in manifest") + } + _, err := a.feed.Lookup(ctx, feed.NewQueryLatest(entry.Feed, lookup.NoClue)) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Debug(fmt.Sprintf("get resource content error: %v", err)) + log.Debug(fmt.Sprintf("get feed update content error: %v", err)) return reader, mimeType, status, nil, err } - - // use this key to retrieve the latest update - params := mru.LookupLatest(rootAddr) - rsrc, err = a.resource.Lookup(ctx, params) + // get the data of the update + _, rsrcData, err := a.feed.GetContent(entry.Feed) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Debug(fmt.Sprintf("get resource content error: %v", err)) + log.Warn(fmt.Sprintf("get feed update content error: %v", err)) return reader, mimeType, status, nil, err } - // if it's multihash, we will transparently serve the content this multihash points to - // \TODO this resolve is rather expensive all in all, review to see if it can be achieved cheaper - if rsrc.Multihash() { + // extract multihash + decodedMultihash, err := multihash.FromMultihash(rsrcData) + if err != nil { + apiGetInvalid.Inc(1) + status = http.StatusUnprocessableEntity + log.Warn("invalid multihash in feed update", "err", err) + return reader, mimeType, status, nil, err + } + manifestAddr = storage.Address(decodedMultihash) + log.Trace("feed update contains multihash", "key", manifestAddr) - // get the data of the update - _, rsrcData, err := a.resource.GetContent(rootAddr) - if err != nil { - apiGetNotFound.Inc(1) - status = http.StatusNotFound - log.Warn(fmt.Sprintf("get resource content error: %v", err)) - return reader, mimeType, status, nil, err - } + // get the manifest the multihash digest points to + trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt) + if err != nil { + apiGetNotFound.Inc(1) + status = http.StatusNotFound + log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err)) + return reader, mimeType, status, nil, err + } - // validate that data as multihash - decodedMultihash, err := multihash.FromMultihash(rsrcData) - if err != nil { - apiGetInvalid.Inc(1) - status = http.StatusUnprocessableEntity - log.Warn("invalid resource multihash", "err", err) - return reader, mimeType, status, nil, err - } - manifestAddr = storage.Address(decodedMultihash) - log.Trace("resource is multihash", "key", manifestAddr) - - // get the manifest the multihash digest points to - trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt) - if err != nil { - apiGetNotFound.Inc(1) - status = http.StatusNotFound - log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err)) - return reader, mimeType, status, nil, err - } - - // finally, get the manifest entry - // it will always be the entry on path "" - entry, _ = trie.getEntry(path) - if entry == nil { - status = http.StatusNotFound - apiGetNotFound.Inc(1) - err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path) - log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path) - return reader, mimeType, status, nil, err - } - - } else { - // data is returned verbatim since it's not a multihash - return rsrc, "application/octet-stream", http.StatusOK, nil, nil + // finally, get the manifest entry + // it will always be the entry on path "" + entry, _ = trie.getEntry(path) + if entry == nil { + status = http.StatusNotFound + apiGetNotFound.Inc(1) + err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path) + log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path) + return reader, mimeType, status, nil, err } } - // regardless of resource update manifests or normal manifests we will converge at this point + // regardless of feed update manifests or normal manifests we will converge at this point // get the key the manifest entry points to and serve it if it's unambiguous contentAddr = common.Hex2Bytes(entry.Hash) status = entry.Status @@ -778,9 +760,14 @@ func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestP // add the entry under the path from the request manifestPath := path.Join(manifestPath, hdr.Name) + contentType := hdr.Xattrs["user.swarm.content-type"] + if contentType == "" { + contentType = mime.TypeByExtension(filepath.Ext(hdr.Name)) + } + //DetectContentType("") entry := &ManifestEntry{ Path: manifestPath, - ContentType: hdr.Xattrs["user.swarm.content-type"], + ContentType: contentType, Mode: hdr.Mode, Size: hdr.Size, ModTime: hdr.ModTime, @@ -791,10 +778,15 @@ func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestP return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err) } if hdr.Name == defaultPath { + contentType := hdr.Xattrs["user.swarm.content-type"] + if contentType == "" { + contentType = mime.TypeByExtension(filepath.Ext(hdr.Name)) + } + entry := &ManifestEntry{ Hash: contentKey.Hex(), Path: "", // default entry - ContentType: hdr.Xattrs["user.swarm.content-type"], + ContentType: contentType, Mode: hdr.Mode, Size: hdr.Size, ModTime: hdr.ModTime, @@ -965,57 +957,120 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver return addr, manifestEntryMap, nil } -// ResourceLookup finds mutable resource updates at specific periods and versions -func (a *API) ResourceLookup(ctx context.Context, params *mru.LookupParams) (string, []byte, error) { - var err error - rsrc, err := a.resource.Load(ctx, params.RootAddr()) +// FeedsLookup finds Swarm feeds updates at specific points in time, or the latest update +func (a *API) FeedsLookup(ctx context.Context, query *feed.Query) ([]byte, error) { + _, err := a.feed.Lookup(ctx, query) if err != nil { - return "", nil, err - } - _, err = a.resource.Lookup(ctx, params) - if err != nil { - return "", nil, err + return nil, err } var data []byte - _, data, err = a.resource.GetContent(params.RootAddr()) + _, data, err = a.feed.GetContent(&query.Feed) if err != nil { - return "", nil, err + return nil, err } - return rsrc.Name(), data, nil + return data, nil } -// Create Mutable resource -func (a *API) ResourceCreate(ctx context.Context, request *mru.Request) error { - return a.resource.New(ctx, request) +// FeedsNewRequest creates a Request object to update a specific feed +func (a *API) FeedsNewRequest(ctx context.Context, feed *feed.Feed) (*feed.Request, error) { + return a.feed.NewRequest(ctx, feed) } -// ResourceNewRequest creates a Request object to update a specific mutable resource -func (a *API) ResourceNewRequest(ctx context.Context, rootAddr storage.Address) (*mru.Request, error) { - return a.resource.NewUpdateRequest(ctx, rootAddr) +// FeedsUpdate publishes a new update on the given feed +func (a *API) FeedsUpdate(ctx context.Context, request *feed.Request) (storage.Address, error) { + return a.feed.Update(ctx, request) } -// ResourceUpdate updates a Mutable Resource with arbitrary data. -// Upon retrieval the update will be retrieved verbatim as bytes. -func (a *API) ResourceUpdate(ctx context.Context, request *mru.SignedResourceUpdate) (storage.Address, error) { - return a.resource.Update(ctx, request) +// FeedsHashSize returned the size of the digest produced by Swarm feeds' hashing function +func (a *API) FeedsHashSize() int { + return a.feed.HashSize } -// ResourceHashSize returned the size of the digest produced by the Mutable Resource hashing function -func (a *API) ResourceHashSize() int { - return a.resource.HashSize -} +// ErrCannotLoadFeedManifest is returned when looking up a feeds manifest fails +var ErrCannotLoadFeedManifest = errors.New("Cannot load feed manifest") -// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk. -func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) { +// ErrNotAFeedManifest is returned when the address provided returned something other than a valid manifest +var ErrNotAFeedManifest = errors.New("Not a feed manifest") + +// ResolveFeedManifest retrieves the Swarm feed manifest for the given address, and returns the referenced Feed. +func (a *API) ResolveFeedManifest(ctx context.Context, addr storage.Address) (*feed.Feed, error) { trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt) if err != nil { - return nil, fmt.Errorf("cannot load resource manifest: %v", err) + return nil, ErrCannotLoadFeedManifest } entry, _ := trie.getEntry("") - if entry.ContentType != ResourceContentType { - return nil, fmt.Errorf("not a resource manifest: %s", addr) + if entry.ContentType != FeedContentType { + return nil, ErrNotAFeedManifest } - return storage.Address(common.FromHex(entry.Hash)), nil + return entry.Feed, nil +} + +// ErrCannotResolveFeedURI is returned when the ENS resolver is not able to translate a name to a Swarm feed +var ErrCannotResolveFeedURI = errors.New("Cannot resolve Feed URI") + +// ErrCannotResolveFeed is returned when values provided are not enough or invalid to recreate a +// feed out of them. +var ErrCannotResolveFeed = errors.New("Cannot resolve Feed") + +// ResolveFeed attempts to extract feed information out of the manifest, if provided +// If not, it attempts to extract the feed out of a set of key-value pairs +func (a *API) ResolveFeed(ctx context.Context, uri *URI, values feed.Values) (*feed.Feed, error) { + var fd *feed.Feed + var err error + if uri.Addr != "" { + // resolve the content key. + manifestAddr := uri.Address() + if manifestAddr == nil { + manifestAddr, err = a.Resolve(ctx, uri.Addr) + if err != nil { + return nil, ErrCannotResolveFeedURI + } + } + + // get the Swarm feed from the manifest + fd, err = a.ResolveFeedManifest(ctx, manifestAddr) + if err != nil { + return nil, err + } + log.Debug("handle.get.feed: resolved", "manifestkey", manifestAddr, "feed", fd.Hex()) + } else { + var f feed.Feed + if err := f.FromValues(values); err != nil { + return nil, ErrCannotResolveFeed + + } + fd = &f + } + return fd, nil +} + +// MimeOctetStream default value of http Content-Type header +const MimeOctetStream = "application/octet-stream" + +// DetectContentType by file file extension, or fallback to content sniff +func DetectContentType(fileName string, f io.ReadSeeker) (string, error) { + ctype := mime.TypeByExtension(filepath.Ext(fileName)) + if ctype != "" { + return ctype, nil + } + + // save/rollback to get content probe from begin of file + currentPosition, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return MimeOctetStream, fmt.Errorf("seeker can't seek, %s", err) + } + + // read a chunk to decide between utf-8 text and binary + var buf [512]byte + n, _ := f.Read(buf[:]) + ctype = http.DetectContentType(buf[:n]) + + _, err = f.Seek(currentPosition, io.SeekStart) // rewind to output whole file + if err != nil { + return MimeOctetStream, fmt.Errorf("seeker can't seek, %s", err) + } + + return ctype, nil } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index a65bf07e2e..eb896f32aa 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -17,6 +17,7 @@ package api import ( + "bytes" "context" "errors" "flag" @@ -433,3 +434,69 @@ func TestDecryptOrigin(t *testing.T) { } } } + +func TestDetectContentType(t *testing.T) { + for _, tc := range []struct { + file string + content string + expectedContentType string + }{ + { + file: "file-with-correct-css.css", + content: "body {background-color: orange}", + expectedContentType: "text/css; charset=utf-8", + }, + { + file: "empty-file.css", + content: "", + expectedContentType: "text/css; charset=utf-8", + }, + { + file: "empty-file.pdf", + content: "", + expectedContentType: "application/pdf", + }, + { + file: "empty-file.md", + content: "", + expectedContentType: "text/markdown; charset=utf-8", + }, + { + file: "empty-file-with-unknown-content.strangeext", + content: "", + expectedContentType: "text/plain; charset=utf-8", + }, + { + file: "file-with-unknown-extension-and-content.strangeext", + content: "Lorem Ipsum", + expectedContentType: "text/plain; charset=utf-8", + }, + { + file: "file-no-extension", + content: "Lorem Ipsum", + expectedContentType: "text/plain; charset=utf-8", + }, + { + file: "file-no-extension-no-content", + content: "", + expectedContentType: "text/plain; charset=utf-8", + }, + { + file: "css-file-with-html-inside.css", + content: "", + expectedContentType: "text/css; charset=utf-8", + }, + } { + t.Run(tc.file, func(t *testing.T) { + detected, err := DetectContentType(tc.file, bytes.NewReader([]byte(tc.content))) + if err != nil { + t.Fatal(err) + } + + if detected != tc.expectedContentType { + t.Fatalf("File: %s, Expected mime type %s, got %s", tc.file, tc.expectedContentType, detected) + } + + }) + } +} diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 3d06e9e1cc..d9837ca732 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -24,10 +24,10 @@ import ( "fmt" "io" "io/ioutil" - "mime" "mime/multipart" "net/http" "net/textproto" + "net/url" "os" "path/filepath" "regexp" @@ -35,7 +35,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" ) var ( @@ -123,10 +123,16 @@ func Open(path string) (*File, error) { f.Close() return nil, err } + + contentType, err := api.DetectContentType(f.Name(), f) + if err != nil { + return nil, err + } + return &File{ ReadCloser: f, ManifestEntry: api.ManifestEntry{ - ContentType: mime.TypeByExtension(filepath.Ext(path)), + ContentType: contentType, Mode: int64(stat.Mode()), Size: stat.Size(), ModTime: stat.ModTime(), @@ -595,13 +601,15 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) return string(data), nil } -// CreateResource creates a Mutable Resource with the given name and frequency, initializing it with the provided -// data. Data is interpreted as multihash or not depending on the multihash parameter. -// startTime=0 means "now" -// Returns the resulting Mutable Resource manifest address that you can use to include in an ENS Resolver (setContent) -// or reference future updates (Client.UpdateResource) -func (c *Client) CreateResource(request *mru.Request) (string, error) { - responseStream, err := c.updateResource(request) +// ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed +var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed") + +// CreateFeedWithManifest creates a feed manifest, initializing it with the provided +// data +// Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent) +// or reference future updates (Client.UpdateFeed) +func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) { + responseStream, err := c.updateFeed(request, true) if err != nil { return "", err } @@ -619,19 +627,26 @@ func (c *Client) CreateResource(request *mru.Request) (string, error) { return manifestAddress, nil } -// UpdateResource allows you to set a new version of your content -func (c *Client) UpdateResource(request *mru.Request) error { - _, err := c.updateResource(request) +// UpdateFeed allows you to set a new version of your content +func (c *Client) UpdateFeed(request *feed.Request) error { + _, err := c.updateFeed(request, false) return err } -func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) { - body, err := request.MarshalJSON() +func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) { + URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } + URL.Path = "/bzz-feed:/" + values := URL.Query() + body := request.AppendValues(values) + if createManifest { + values.Set("manifest", "1") + } + URL.RawQuery = values.Encode() - req, err := http.NewRequest("POST", c.Gateway+"/bzz-resource:/", bytes.NewBuffer(body)) + req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body)) if err != nil { return nil, err } @@ -642,28 +657,61 @@ func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) { } return res.Body, nil - } -// GetResource returns a byte stream with the raw content of the resource -// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver +// QueryFeed returns a byte stream with the raw content of the feed update +// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address -func (c *Client) GetResource(manifestAddressOrDomain string) (io.ReadCloser, error) { +func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { + return c.queryFeed(query, manifestAddressOrDomain, false) +} - res, err := http.Get(c.Gateway + "/bzz-resource:/" + manifestAddressOrDomain) +// queryFeed returns a byte stream with the raw content of the feed update +// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver +// points to that address +// meta set to true will instruct the node return feed metainformation instead +func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { + URL, err := url.Parse(c.Gateway) + if err != nil { + return nil, err + } + URL.Path = "/bzz-feed:/" + manifestAddressOrDomain + values := URL.Query() + if query != nil { + query.AppendValues(values) //adds query parameters + } + if meta { + values.Set("meta", "1") + } + URL.RawQuery = values.Encode() + res, err := http.Get(URL.String()) if err != nil { return nil, err } - return res.Body, nil + if res.StatusCode != http.StatusOK { + if res.StatusCode == http.StatusNotFound { + return nil, ErrNoFeedUpdatesFound + } + errorMessageBytes, err := ioutil.ReadAll(res.Body) + var errorMessage string + if err != nil { + errorMessage = "cannot retrieve error message: " + err.Error() + } else { + errorMessage = string(errorMessageBytes) + } + return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage) + } + + return res.Body, nil } -// GetResourceMetadata returns a structure that describes the Mutable Resource -// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver +// GetFeedRequest returns a structure that describes the referenced feed status +// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address -func (c *Client) GetResourceMetadata(manifestAddressOrDomain string) (*mru.Request, error) { +func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) { - responseStream, err := c.GetResource(manifestAddressOrDomain + "/meta") + responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true) if err != nil { return nil, err } @@ -674,7 +722,7 @@ func (c *Client) GetResourceMetadata(manifestAddressOrDomain string) (*mru.Reque return nil, err } - var metadata mru.Request + var metadata feed.Request if err := metadata.UnmarshalJSON(body); err != nil { return nil, err } diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index f9312d48fe..76b3493978 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -25,16 +25,17 @@ import ( "sort" "testing" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/multihash" - "github.com/ethereum/go-ethereum/swarm/storage/mru" - "github.com/ethereum/go-ethereum/swarm/testutil" + "github.com/ethereum/go-ethereum/swarm/storage/feed" ) -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) swarmhttp.TestServer { return swarmhttp.NewServer(api, "") } @@ -47,7 +48,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) { } func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -88,7 +89,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) { } func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -186,7 +187,7 @@ func newTestDirectory(t *testing.T) string { // TestClientUploadDownloadDirectory tests uploading and downloading a // directory of files to a swarm manifest func TestClientUploadDownloadDirectory(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -252,7 +253,7 @@ func TestClientFileListEncrypted(t *testing.T) { } func testClientFileList(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -310,7 +311,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { // TestClientMultipartUpload tests uploading files to swarm using a multipart // upload func TestClientMultipartUpload(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // define an uploader which uploads testDirFiles with some data @@ -359,24 +360,24 @@ func TestClientMultipartUpload(t *testing.T) { } } -func newTestSigner() (*mru.GenericSigner, error) { +func newTestSigner() (*feed.GenericSigner, error) { privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") if err != nil { return nil, err } - return mru.NewGenericSigner(privKey), nil + return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash resource types with bzz:// scheme +// test the transparent resolving of multihash feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a resource update +// first upload data, and store the multihash to the resulting manifest in a feed update // retrieving the update with the multihash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestClientCreateResourceMultihash(t *testing.T) { +func TestClientCreateFeedMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() @@ -391,37 +392,36 @@ func TestClientCreateResourceMultihash(t *testing.T) { s := common.FromHex(swarmHash) mh := multihash.ToMultihash(s) - // our mutable resource "name" - resourceName := "foo.eth" + // our feed topic + topic, _ := feed.NewTopic("foo.eth", nil) - createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{ - Name: resourceName, - Frequency: 13, - StartTime: srv.GetCurrentTime(), - Owner: signer.Address(), - }) - if err != nil { - t.Fatal(err) - } - createRequest.SetData(mh, true) + createRequest := feed.NewFirstRequest(topic) + + createRequest.SetData(mh) if err := createRequest.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - resourceManifestHash, err := client.CreateResource(createRequest) + feedManifestHash, err := client.CreateFeedWithManifest(createRequest) if err != nil { - t.Fatalf("Error creating resource: %s", err) + t.Fatalf("Error creating feed manifest: %s", err) } - correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e" - if resourceManifestHash != correctManifestAddrHex { - t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash) + correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" + if feedManifestHash != correctManifestAddrHex { + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) } - reader, err := client.GetResource(correctManifestAddrHex) + // Check we get a not found error when trying to get feed updates with a made-up manifest + _, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + if err != ErrNoFeedUpdatesFound { + t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err) + } + + reader, err := client.QueryFeed(nil, correctManifestAddrHex) if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err := ioutil.ReadAll(reader) @@ -434,45 +434,40 @@ func TestClientCreateResourceMultihash(t *testing.T) { } -// TestClientCreateUpdateResource will check that mutable resources can be created and updated via the HTTP client. -func TestClientCreateUpdateResource(t *testing.T) { +// TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client. +func TestClientCreateUpdateFeed(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() - // set raw data for the resource + // set raw data for the feed update databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...") - // our mutable resource name - resourceName := "El Quijote" + // our feed topic name + topic, _ := feed.NewTopic("El Quijote", nil) + createRequest := feed.NewFirstRequest(topic) - createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{ - Name: resourceName, - Frequency: 13, - StartTime: srv.GetCurrentTime(), - Owner: signer.Address(), - }) - if err != nil { - t.Fatal(err) - } - createRequest.SetData(databytes, false) + createRequest.SetData(databytes) if err := createRequest.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - resourceManifestHash, err := client.CreateResource(createRequest) - - correctManifestAddrHex := "cc7904c17b49f9679e2d8006fe25e87e3f5c2072c2b49cab50f15e544471b30a" - if resourceManifestHash != correctManifestAddrHex { - t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash) + feedManifestHash, err := client.CreateFeedWithManifest(createRequest) + if err != nil { + t.Fatal(err) } - reader, err := client.GetResource(correctManifestAddrHex) + correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882" + if feedManifestHash != correctManifestAddrHex { + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) + } + + reader, err := client.QueryFeed(nil, correctManifestAddrHex) if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err := ioutil.ReadAll(reader) @@ -486,23 +481,23 @@ func TestClientCreateUpdateResource(t *testing.T) { // define different data databytes = []byte("... no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero ...") - updateRequest, err := client.GetResourceMetadata(correctManifestAddrHex) + updateRequest, err := client.GetFeedRequest(nil, correctManifestAddrHex) if err != nil { t.Fatalf("Error retrieving update request template: %s", err) } - updateRequest.SetData(databytes, false) + updateRequest.SetData(databytes) if err := updateRequest.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - if err = client.UpdateResource(updateRequest); err != nil { - t.Fatalf("Error updating resource: %s", err) + if err = client.UpdateFeed(updateRequest); err != nil { + t.Fatalf("Error updating feed: %s", err) } - reader, err = client.GetResource(correctManifestAddrHex) + reader, err = client.QueryFeed(nil, correctManifestAddrHex) if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err = ioutil.ReadAll(reader) @@ -513,4 +508,24 @@ func TestClientCreateUpdateResource(t *testing.T) { t.Fatalf("Expected: %v, got %v", databytes, gotData) } + // now try retrieving feed updates without a manifest + + fd := &feed.Feed{ + Topic: topic, + User: signer.Address(), + } + + lookupParams := feed.NewQueryLatest(fd, lookup.NoClue) + reader, err = client.QueryFeed(lookupParams, "") + if err != nil { + t.Fatalf("Error retrieving feed updates: %s", err) + } + defer reader.Close() + gotData, err = ioutil.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(databytes, gotData) { + t.Fatalf("Expected: %v, got %v", databytes, gotData) + } } diff --git a/swarm/api/config.go b/swarm/api/config.go index e753890e49..be73854087 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -50,26 +50,27 @@ type Config struct { Swap *swap.LocalProfile Pss *pss.PssParams //*network.SyncParams - Contract common.Address - EnsRoot common.Address - EnsAPIs []string - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - NodeID string - NetworkID uint64 - SwapEnabled bool - SyncEnabled bool - SyncingSkipCheck bool - DeliverySkipCheck bool - LightNodeEnabled bool - SyncUpdateDelay time.Duration - SwapAPI string - Cors string - BzzAccount string - privateKey *ecdsa.PrivateKey + Contract common.Address + EnsRoot common.Address + EnsAPIs []string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NodeID string + NetworkID uint64 + SwapEnabled bool + SyncEnabled bool + SyncingSkipCheck bool + DeliverySkipCheck bool + MaxStreamPeerServers int + LightNodeEnabled bool + SyncUpdateDelay time.Duration + SwapAPI string + Cors string + BzzAccount string + privateKey *ecdsa.PrivateKey } //create a default config with all parameters to set to defaults @@ -80,20 +81,21 @@ func NewConfig() (c *Config) { FileStoreParams: storage.NewFileStoreParams(), HiveParams: network.NewHiveParams(), //SyncParams: network.NewDefaultSyncParams(), - Swap: swap.NewDefaultSwapParams(), - Pss: pss.NewPssParams(), - ListenAddr: DefaultHTTPListenAddr, - Port: DefaultHTTPPort, - Path: node.DefaultDataDir(), - EnsAPIs: nil, - EnsRoot: ens.TestNetAddress, - NetworkID: network.DefaultNetworkID, - SwapEnabled: false, - SyncEnabled: true, - SyncingSkipCheck: false, - DeliverySkipCheck: true, - SyncUpdateDelay: 15 * time.Second, - SwapAPI: "", + Swap: swap.NewDefaultSwapParams(), + Pss: pss.NewPssParams(), + ListenAddr: DefaultHTTPListenAddr, + Port: DefaultHTTPPort, + Path: node.DefaultDataDir(), + EnsAPIs: nil, + EnsRoot: ens.TestNetAddress, + NetworkID: network.DefaultNetworkID, + SwapEnabled: false, + SyncEnabled: true, + SyncingSkipCheck: false, + MaxStreamPeerServers: 10000, + DeliverySkipCheck: true, + SyncUpdateDelay: 15 * time.Second, + SwapAPI: "", } return diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 8251ebc4dc..266ef71bec 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "io" - "net/http" "os" "path" "path/filepath" @@ -97,51 +96,54 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error list = append(list, entry) } - cnt := len(list) - errors := make([]error, cnt) - done := make(chan bool, maxParallelFiles) - dcnt := 0 - awg := &sync.WaitGroup{} + errors := make([]error, len(list)) + sem := make(chan bool, maxParallelFiles) + defer close(sem) for i, entry := range list { - if i >= dcnt+maxParallelFiles { - <-done - dcnt++ - } - awg.Add(1) - go func(i int, entry *manifestTrieEntry, done chan bool) { + sem <- true + go func(i int, entry *manifestTrieEntry) { + defer func() { <-sem }() + f, err := os.Open(entry.Path) - if err == nil { - stat, _ := f.Stat() - var hash storage.Address - var wait func(context.Context) error - ctx := context.TODO() - hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt) - if hash != nil { - list[i].Hash = hash.Hex() - } - err = wait(ctx) - awg.Done() - if err == nil { - first512 := make([]byte, 512) - fread, _ := f.ReadAt(first512, 0) - if fread > 0 { - mimeType := http.DetectContentType(first512[:fread]) - if filepath.Ext(entry.Path) == ".css" { - mimeType = "text/css" - } - list[i].ContentType = mimeType - } - } - f.Close() + if err != nil { + errors[i] = err + return } - errors[i] = err - done <- true - }(i, entry, done) + defer f.Close() + + stat, err := f.Stat() + if err != nil { + errors[i] = err + return + } + + var hash storage.Address + var wait func(context.Context) error + ctx := context.TODO() + hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt) + if err != nil { + errors[i] = err + return + } + if hash != nil { + list[i].Hash = hash.Hex() + } + if err := wait(ctx); err != nil { + errors[i] = err + return + } + + list[i].ContentType, err = DetectContentType(f.Name(), f) + if err != nil { + errors[i] = err + return + } + + }(i, entry) } - for dcnt < cnt { - <-done - dcnt++ + for i := 0; i < cap(sem); i++ { + sem <- true } trie := &manifestTrie{ @@ -168,7 +170,6 @@ func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error if err2 == nil { hs = trie.ref.Hex() } - awg.Wait() return hs, err2 } diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index fe7527b1f1..02f5bff658 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -60,7 +60,7 @@ func TestApiDirUpload0(t *testing.T) { content = readPath(t, "testdata", "test0", "index.css") resp = testGet(t, api, bzzhash, "index.css") - exp = expResponse(content, "text/css", 0) + exp = expResponse(content, "text/css; charset=utf-8", 0) checkResponse(t, resp, exp) addr := storage.Address(common.Hex2Bytes(bzzhash)) @@ -140,7 +140,7 @@ func TestApiDirUploadModify(t *testing.T) { content = readPath(t, "testdata", "test0", "index.css") resp = testGet(t, api, bzzhash, "index.css") - exp = expResponse(content, "text/css", 0) + exp = expResponse(content, "text/css; charset=utf-8", 0) checkResponse(t, resp, exp) _, _, _, _, err = api.Get(context.TODO(), nil, addr, "") diff --git a/swarm/api/gen_mime.go b/swarm/api/gen_mime.go new file mode 100644 index 0000000000..109edeb506 --- /dev/null +++ b/swarm/api/gen_mime.go @@ -0,0 +1,1201 @@ +// Code generated by github.com/ethereum/go-ethereum/cmd/swarm/mimegen. DO NOT EDIT. + +package api + +import "mime" + +func init() { + var mimeTypes = map[string]string{ + ".a2l": "application/A2L", + ".aml": "application/AML", + ".ez": "application/andrew-inset", + ".atf": "application/ATF", + ".atfx": "application/ATFX", + ".atxml": "application/ATXML", + ".atom": "application/atom+xml", + ".atomcat": "application/atomcat+xml", + ".atomdeleted": "application/atomdeleted+xml", + ".atomsvc": "application/atomsvc+xml", + ".apxml": "application/auth-policy+xml", + ".xdd": "application/bacnet-xdd+zip", + ".xcs": "application/calendar+xml", + ".cbor": "application/cbor", + ".ccmp": "application/ccmp+xml", + ".ccxml": "application/ccxml+xml", + ".cdfx": "application/CDFX+XML", + ".cdmia": "application/cdmi-capability", + ".cdmic": "application/cdmi-container", + ".cdmid": "application/cdmi-domain", + ".cdmio": "application/cdmi-object", + ".cdmiq": "application/cdmi-queue", + ".cea": "application/CEA", + ".cellml": "application/cellml+xml", + ".cml": "application/cellml+xml", + ".clue": "application/clue_info+xml", + ".cmsc": "application/cms", + ".cpl": "application/cpl+xml", + ".csrattrs": "application/csrattrs", + ".mpd": "application/dash+xml", + ".mpdd": "application/dashdelta", + ".davmount": "application/davmount+xml", + ".dcd": "application/DCD", + ".dcm": "application/dicom", + ".dii": "application/DII", + ".dit": "application/DIT", + ".xmls": "application/dskpp+xml", + ".dssc": "application/dssc+der", + ".xdssc": "application/dssc+xml", + ".dvc": "application/dvcs", + ".es": "application/ecmascript", + ".efi": "application/efi", + ".emma": "application/emma+xml", + ".emotionml": "application/emotionml+xml", + ".epub": "application/epub+zip", + ".exi": "application/exi", + ".finf": "application/fastinfoset", + ".fdt": "application/fdt+xml", + ".pfr": "application/font-tdpfr", + ".geojson": "application/geo+json", + ".gml": "application/gml+xml", + ".gz": "application/gzip", + ".tgz": "application/gzip", + ".stk": "application/hyperstudio", + ".ink": "application/inkml+xml", + ".inkml": "application/inkml+xml", + ".ipfix": "application/ipfix", + ".its": "application/its+xml", + ".js": "application/javascript", + ".jrd": "application/jrd+json", + ".json": "application/json", + ".json-patch": "application/json-patch+json", + ".jsonld": "application/ld+json", + ".lgr": "application/lgr+xml", + ".wlnk": "application/link-format", + ".lostxml": "application/lost+xml", + ".lostsyncxml": "application/lostsync+xml", + ".lxf": "application/LXF", + ".hqx": "application/mac-binhex40", + ".mads": "application/mads+xml", + ".mrc": "application/marc", + ".mrcx": "application/marcxml+xml", + ".nb": "application/mathematica", + ".ma": "application/mathematica", + ".mb": "application/mathematica", + ".mml": "application/mathml+xml", + ".mbox": "application/mbox", + ".meta4": "application/metalink4+xml", + ".mets": "application/mets+xml", + ".mf4": "application/MF4", + ".mods": "application/mods+xml", + ".m21": "application/mp21", + ".mp21": "application/mp21", + ".doc": "application/msword", + ".mxf": "application/mxf", + ".nq": "application/n-quads", + ".nt": "application/n-triples", + ".orq": "application/ocsp-request", + ".ors": "application/ocsp-response", + ".bin": "application/octet-stream", + ".lha": "application/octet-stream", + ".lzh": "application/octet-stream", + ".exe": "application/octet-stream", + ".class": "application/octet-stream", + ".so": "application/octet-stream", + ".dll": "application/octet-stream", + ".img": "application/octet-stream", + ".iso": "application/octet-stream", + ".oda": "application/oda", + ".odx": "application/ODX", + ".opf": "application/oebps-package+xml", + ".ogx": "application/ogg", + ".oxps": "application/oxps", + ".relo": "application/p2p-overlay+xml", + ".pdf": "application/pdf", + ".pdx": "application/PDX", + ".pgp": "application/pgp-encrypted", + ".sig": "application/pgp-signature", + ".p10": "application/pkcs10", + ".p12": "application/pkcs12", + ".pfx": "application/pkcs12", + ".p7m": "application/pkcs7-mime", + ".p7c": "application/pkcs7-mime", + ".p7s": "application/pkcs7-signature", + ".p8": "application/pkcs8", + ".cer": "application/pkix-cert", + ".crl": "application/pkix-crl", + ".pkipath": "application/pkix-pkipath", + ".pki": "application/pkixcmp", + ".pls": "application/pls+xml", + ".ps": "application/postscript", + ".eps": "application/postscript", + ".ai": "application/postscript", + ".provx": "application/provenance+xml", + ".cw": "application/prs.cww", + ".cww": "application/prs.cww", + ".hpub": "application/prs.hpub+zip", + ".rnd": "application/prs.nprend", + ".rct": "application/prs.nprend", + ".rdf-crypt": "application/prs.rdf-xml-crypt", + ".xsf": "application/prs.xsf+xml", + ".pskcxml": "application/pskc+xml", + ".rdf": "application/rdf+xml", + ".rif": "application/reginfo+xml", + ".rnc": "application/relax-ng-compact-syntax", + ".rld": "application/resource-lists-diff+xml", + ".rl": "application/resource-lists+xml", + ".rfcxml": "application/rfc+xml", + ".rs": "application/rls-services+xml", + ".gbr": "application/rpki-ghostbusters", + ".mft": "application/rpki-manifest", + ".roa": "application/rpki-roa", + ".rtf": "application/rtf", + ".scim": "application/scim+json", + ".scq": "application/scvp-cv-request", + ".scs": "application/scvp-cv-response", + ".spq": "application/scvp-vp-request", + ".spp": "application/scvp-vp-response", + ".sdp": "application/sdp", + ".soc": "application/sgml-open-catalog", + ".shf": "application/shf+xml", + ".siv": "application/sieve", + ".sieve": "application/sieve", + ".cl": "application/simple-filter+xml", + ".smil": "application/smil+xml", + ".smi": "application/smil+xml", + ".sml": "application/smil+xml", + ".rq": "application/sparql-query", + ".srx": "application/sparql-results+xml", + ".sql": "application/sql", + ".gram": "application/srgs", + ".grxml": "application/srgs+xml", + ".sru": "application/sru+xml", + ".ssml": "application/ssml+xml", + ".tau": "application/tamp-apex-update", + ".auc": "application/tamp-apex-update-confirm", + ".tcu": "application/tamp-community-update", + ".cuc": "application/tamp-community-update-confirm", + ".ter": "application/tamp-error", + ".tsa": "application/tamp-sequence-adjust", + ".sac": "application/tamp-sequence-adjust-confirm", + ".tur": "application/tamp-update", + ".tuc": "application/tamp-update-confirm", + ".tei": "application/tei+xml", + ".teiCorpus": "application/tei+xml", + ".odd": "application/tei+xml", + ".tfi": "application/thraud+xml", + ".tsq": "application/timestamp-query", + ".tsr": "application/timestamp-reply", + ".tsd": "application/timestamped-data", + ".trig": "application/trig", + ".ttml": "application/ttml+xml", + ".gsheet": "application/urc-grpsheet+xml", + ".rsheet": "application/urc-ressheet+xml", + ".td": "application/urc-targetdesc+xml", + ".uis": "application/urc-uisocketdesc+xml", + ".plb": "application/vnd.3gpp.pic-bw-large", + ".psb": "application/vnd.3gpp.pic-bw-small", + ".pvb": "application/vnd.3gpp.pic-bw-var", + ".sms": "application/vnd.3gpp2.sms", + ".tcap": "application/vnd.3gpp2.tcap", + ".imgcal": "application/vnd.3lightssoftware.imagescal", + ".pwn": "application/vnd.3M.Post-it-Notes", + ".aso": "application/vnd.accpac.simply.aso", + ".imp": "application/vnd.accpac.simply.imp", + ".acu": "application/vnd.acucobol", + ".atc": "application/vnd.acucorp", + ".acutc": "application/vnd.acucorp", + ".swf": "application/vnd.adobe.flash.movie", + ".fcdt": "application/vnd.adobe.formscentral.fcdt", + ".fxp": "application/vnd.adobe.fxp", + ".fxpl": "application/vnd.adobe.fxp", + ".xdp": "application/vnd.adobe.xdp+xml", + ".xfdf": "application/vnd.adobe.xfdf", + ".ahead": "application/vnd.ahead.space", + ".azf": "application/vnd.airzip.filesecure.azf", + ".azs": "application/vnd.airzip.filesecure.azs", + ".azw3": "application/vnd.amazon.mobi8-ebook", + ".acc": "application/vnd.americandynamics.acc", + ".ami": "application/vnd.amiga.ami", + ".apkg": "application/vnd.anki", + ".cii": "application/vnd.anser-web-certificate-issue-initiation", + ".fti": "application/vnd.anser-web-funds-transfer-initiation", + ".dist": "application/vnd.apple.installer+xml", + ".distz": "application/vnd.apple.installer+xml", + ".pkg": "application/vnd.apple.installer+xml", + ".mpkg": "application/vnd.apple.installer+xml", + ".m3u8": "application/vnd.apple.mpegurl", + ".swi": "application/vnd.aristanetworks.swi", + ".iota": "application/vnd.astraea-software.iota", + ".aep": "application/vnd.audiograph", + ".package": "application/vnd.autopackage", + ".bmml": "application/vnd.balsamiq.bmml+xml", + ".bmpr": "application/vnd.balsamiq.bmpr", + ".mpm": "application/vnd.blueice.multipass", + ".ep": "application/vnd.bluetooth.ep.oob", + ".le": "application/vnd.bluetooth.le.oob", + ".bmi": "application/vnd.bmi", + ".rep": "application/vnd.businessobjects", + ".tlclient": "application/vnd.cendio.thinlinc.clientconf", + ".cdxml": "application/vnd.chemdraw+xml", + ".pgn": "application/vnd.chess-pgn", + ".mmd": "application/vnd.chipnuts.karaoke-mmd", + ".cdy": "application/vnd.cinderella", + ".csl": "application/vnd.citationstyles.style+xml", + ".cla": "application/vnd.claymore", + ".rp9": "application/vnd.cloanto.rp9", + ".c4g": "application/vnd.clonk.c4group", + ".c4d": "application/vnd.clonk.c4group", + ".c4f": "application/vnd.clonk.c4group", + ".c4p": "application/vnd.clonk.c4group", + ".c4u": "application/vnd.clonk.c4group", + ".c11amc": "application/vnd.cluetrust.cartomobile-config", + ".c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", + ".coffee": "application/vnd.coffeescript", + ".cbz": "application/vnd.comicbook+zip", + ".ica": "application/vnd.commerce-battelle", + ".icf": "application/vnd.commerce-battelle", + ".icd": "application/vnd.commerce-battelle", + ".ic0": "application/vnd.commerce-battelle", + ".ic1": "application/vnd.commerce-battelle", + ".ic2": "application/vnd.commerce-battelle", + ".ic3": "application/vnd.commerce-battelle", + ".ic4": "application/vnd.commerce-battelle", + ".ic5": "application/vnd.commerce-battelle", + ".ic6": "application/vnd.commerce-battelle", + ".ic7": "application/vnd.commerce-battelle", + ".ic8": "application/vnd.commerce-battelle", + ".csp": "application/vnd.commonspace", + ".cst": "application/vnd.commonspace", + ".cdbcmsg": "application/vnd.contact.cmsg", + ".ign": "application/vnd.coreos.ignition+json", + ".ignition": "application/vnd.coreos.ignition+json", + ".cmc": "application/vnd.cosmocaller", + ".clkx": "application/vnd.crick.clicker", + ".clkk": "application/vnd.crick.clicker.keyboard", + ".clkp": "application/vnd.crick.clicker.palette", + ".clkt": "application/vnd.crick.clicker.template", + ".clkw": "application/vnd.crick.clicker.wordbank", + ".wbs": "application/vnd.criticaltools.wbs+xml", + ".pml": "application/vnd.ctc-posml", + ".ppd": "application/vnd.cups-ppd", + ".curl": "application/vnd.curl", + ".dart": "application/vnd.dart", + ".rdz": "application/vnd.data-vision.rdz", + ".deb": "application/vnd.debian.binary-package", + ".udeb": "application/vnd.debian.binary-package", + ".uvf": "application/vnd.dece.data", + ".uvvf": "application/vnd.dece.data", + ".uvd": "application/vnd.dece.data", + ".uvvd": "application/vnd.dece.data", + ".uvt": "application/vnd.dece.ttml+xml", + ".uvvt": "application/vnd.dece.ttml+xml", + ".uvx": "application/vnd.dece.unspecified", + ".uvvx": "application/vnd.dece.unspecified", + ".uvz": "application/vnd.dece.zip", + ".uvvz": "application/vnd.dece.zip", + ".fe_launch": "application/vnd.denovo.fcselayout-link", + ".dsm": "application/vnd.desmume.movie", + ".dna": "application/vnd.dna", + ".docjson": "application/vnd.document+json", + ".scld": "application/vnd.doremir.scorecloud-binary-document", + ".dpg": "application/vnd.dpgraph", + ".mwc": "application/vnd.dpgraph", + ".dpgraph": "application/vnd.dpgraph", + ".dfac": "application/vnd.dreamfactory", + ".fla": "application/vnd.dtg.local.flash", + ".ait": "application/vnd.dvb.ait", + ".svc": "application/vnd.dvb.service", + ".geo": "application/vnd.dynageo", + ".dzr": "application/vnd.dzr", + ".mag": "application/vnd.ecowin.chart", + ".nml": "application/vnd.enliven", + ".esf": "application/vnd.epson.esf", + ".msf": "application/vnd.epson.msf", + ".qam": "application/vnd.epson.quickanime", + ".slt": "application/vnd.epson.salt", + ".ssf": "application/vnd.epson.ssf", + ".qcall": "application/vnd.ericsson.quickcall", + ".qca": "application/vnd.ericsson.quickcall", + ".espass": "application/vnd.espass-espass+zip", + ".es3": "application/vnd.eszigno3+xml", + ".et3": "application/vnd.eszigno3+xml", + ".asice": "application/vnd.etsi.asic-e+zip", + ".sce": "application/vnd.etsi.asic-e+zip", + ".asics": "application/vnd.etsi.asic-s+zip", + ".tst": "application/vnd.etsi.timestamp-token", + ".ez2": "application/vnd.ezpix-album", + ".ez3": "application/vnd.ezpix-package", + ".dim": "application/vnd.fastcopy-disk-image", + ".fdf": "application/vnd.fdf", + ".msd": "application/vnd.fdsn.mseed", + ".mseed": "application/vnd.fdsn.mseed", + ".seed": "application/vnd.fdsn.seed", + ".dataless": "application/vnd.fdsn.seed", + ".zfc": "application/vnd.filmit.zfc", + ".gph": "application/vnd.FloGraphIt", + ".ftc": "application/vnd.fluxtime.clip", + ".sfd": "application/vnd.font-fontforge-sfd", + ".fm": "application/vnd.framemaker", + ".fnc": "application/vnd.frogans.fnc", + ".ltf": "application/vnd.frogans.ltf", + ".fsc": "application/vnd.fsc.weblaunch", + ".oas": "application/vnd.fujitsu.oasys", + ".oa2": "application/vnd.fujitsu.oasys2", + ".oa3": "application/vnd.fujitsu.oasys3", + ".fg5": "application/vnd.fujitsu.oasysgp", + ".bh2": "application/vnd.fujitsu.oasysprs", + ".ddd": "application/vnd.fujixerox.ddd", + ".xdw": "application/vnd.fujixerox.docuworks", + ".xbd": "application/vnd.fujixerox.docuworks.binder", + ".xct": "application/vnd.fujixerox.docuworks.container", + ".fzs": "application/vnd.fuzzysheet", + ".txd": "application/vnd.genomatix.tuxedo", + ".g3": "application/vnd.geocube+xml", + ".g³": "application/vnd.geocube+xml", + ".ggb": "application/vnd.geogebra.file", + ".ggt": "application/vnd.geogebra.tool", + ".gex": "application/vnd.geometry-explorer", + ".gre": "application/vnd.geometry-explorer", + ".gxt": "application/vnd.geonext", + ".g2w": "application/vnd.geoplan", + ".g3w": "application/vnd.geospace", + ".gmx": "application/vnd.gmx", + ".kml": "application/vnd.google-earth.kml+xml", + ".kmz": "application/vnd.google-earth.kmz", + ".gqf": "application/vnd.grafeq", + ".gqs": "application/vnd.grafeq", + ".gac": "application/vnd.groove-account", + ".ghf": "application/vnd.groove-help", + ".gim": "application/vnd.groove-identity-message", + ".grv": "application/vnd.groove-injector", + ".gtm": "application/vnd.groove-tool-message", + ".tpl": "application/vnd.groove-tool-template", + ".vcg": "application/vnd.groove-vcard", + ".hal": "application/vnd.hal+xml", + ".zmm": "application/vnd.HandHeld-Entertainment+xml", + ".hbci": "application/vnd.hbci", + ".hbc": "application/vnd.hbci", + ".kom": "application/vnd.hbci", + ".upa": "application/vnd.hbci", + ".pkd": "application/vnd.hbci", + ".bpd": "application/vnd.hbci", + ".hdt": "application/vnd.hdt", + ".les": "application/vnd.hhe.lesson-player", + ".hpgl": "application/vnd.hp-HPGL", + ".hpi": "application/vnd.hp-hpid", + ".hpid": "application/vnd.hp-hpid", + ".hps": "application/vnd.hp-hps", + ".jlt": "application/vnd.hp-jlyt", + ".pcl": "application/vnd.hp-PCL", + ".sfd-hdstx": "application/vnd.hydrostatix.sof-data", + ".x3d": "application/vnd.hzn-3d-crossword", + ".emm": "application/vnd.ibm.electronic-media", + ".mpy": "application/vnd.ibm.MiniPay", + ".list3820": "application/vnd.ibm.modcap", + ".listafp": "application/vnd.ibm.modcap", + ".afp": "application/vnd.ibm.modcap", + ".pseg3820": "application/vnd.ibm.modcap", + ".irm": "application/vnd.ibm.rights-management", + ".sc": "application/vnd.ibm.secure-container", + ".icc": "application/vnd.iccprofile", + ".icm": "application/vnd.iccprofile", + ".1905.1": "application/vnd.ieee.1905", + ".igl": "application/vnd.igloader", + ".imf": "application/vnd.imagemeter.folder+zip", + ".imi": "application/vnd.imagemeter.image+zip", + ".ivp": "application/vnd.immervision-ivp", + ".ivu": "application/vnd.immervision-ivu", + ".imscc": "application/vnd.ims.imsccv1p1", + ".igm": "application/vnd.insors.igm", + ".xpw": "application/vnd.intercon.formnet", + ".xpx": "application/vnd.intercon.formnet", + ".i2g": "application/vnd.intergeo", + ".qbo": "application/vnd.intu.qbo", + ".qfx": "application/vnd.intu.qfx", + ".rcprofile": "application/vnd.ipunplugged.rcprofile", + ".irp": "application/vnd.irepository.package+xml", + ".xpr": "application/vnd.is-xpr", + ".fcs": "application/vnd.isac.fcs", + ".jam": "application/vnd.jam", + ".rms": "application/vnd.jcp.javame.midlet-rms", + ".jisp": "application/vnd.jisp", + ".joda": "application/vnd.joost.joda-archive", + ".ktz": "application/vnd.kahootz", + ".ktr": "application/vnd.kahootz", + ".karbon": "application/vnd.kde.karbon", + ".chrt": "application/vnd.kde.kchart", + ".kfo": "application/vnd.kde.kformula", + ".flw": "application/vnd.kde.kivio", + ".kon": "application/vnd.kde.kontour", + ".kpr": "application/vnd.kde.kpresenter", + ".kpt": "application/vnd.kde.kpresenter", + ".ksp": "application/vnd.kde.kspread", + ".kwd": "application/vnd.kde.kword", + ".kwt": "application/vnd.kde.kword", + ".htke": "application/vnd.kenameaapp", + ".kia": "application/vnd.kidspiration", + ".kne": "application/vnd.Kinar", + ".knp": "application/vnd.Kinar", + ".sdf": "application/vnd.Kinar", + ".skp": "application/vnd.koan", + ".skd": "application/vnd.koan", + ".skm": "application/vnd.koan", + ".skt": "application/vnd.koan", + ".sse": "application/vnd.kodak-descriptor", + ".lasjson": "application/vnd.las.las+json", + ".lasxml": "application/vnd.las.las+xml", + ".lbd": "application/vnd.llamagraphics.life-balance.desktop", + ".lbe": "application/vnd.llamagraphics.life-balance.exchange+xml", + ".123": "application/vnd.lotus-1-2-3", + ".wk4": "application/vnd.lotus-1-2-3", + ".wk3": "application/vnd.lotus-1-2-3", + ".wk1": "application/vnd.lotus-1-2-3", + ".apr": "application/vnd.lotus-approach", + ".vew": "application/vnd.lotus-approach", + ".prz": "application/vnd.lotus-freelance", + ".pre": "application/vnd.lotus-freelance", + ".nsf": "application/vnd.lotus-notes", + ".ntf": "application/vnd.lotus-notes", + ".ndl": "application/vnd.lotus-notes", + ".ns4": "application/vnd.lotus-notes", + ".ns3": "application/vnd.lotus-notes", + ".ns2": "application/vnd.lotus-notes", + ".nsh": "application/vnd.lotus-notes", + ".nsg": "application/vnd.lotus-notes", + ".or3": "application/vnd.lotus-organizer", + ".or2": "application/vnd.lotus-organizer", + ".org": "application/vnd.lotus-organizer", + ".scm": "application/vnd.lotus-screencam", + ".lwp": "application/vnd.lotus-wordpro", + ".sam": "application/vnd.lotus-wordpro", + ".portpkg": "application/vnd.macports.portpkg", + ".mvt": "application/vnd.mapbox-vector-tile", + ".mdc": "application/vnd.marlin.drm.mdcf", + ".mmdb": "application/vnd.maxmind.maxmind-db", + ".mcd": "application/vnd.mcd", + ".mc1": "application/vnd.medcalcdata", + ".cdkey": "application/vnd.mediastation.cdkey", + ".mwf": "application/vnd.MFER", + ".mfm": "application/vnd.mfmp", + ".flo": "application/vnd.micrografx.flo", + ".igx": "application/vnd.micrografx.igx", + ".mif": "application/vnd.mif", + ".daf": "application/vnd.Mobius.DAF", + ".dis": "application/vnd.Mobius.DIS", + ".mbk": "application/vnd.Mobius.MBK", + ".mqy": "application/vnd.Mobius.MQY", + ".msl": "application/vnd.Mobius.MSL", + ".plc": "application/vnd.Mobius.PLC", + ".txf": "application/vnd.Mobius.TXF", + ".mpn": "application/vnd.mophun.application", + ".mpc": "application/vnd.mophun.certificate", + ".xul": "application/vnd.mozilla.xul+xml", + ".3mf": "application/vnd.ms-3mfdocument", + ".cil": "application/vnd.ms-artgalry", + ".asf": "application/vnd.ms-asf", + ".cab": "application/vnd.ms-cab-compressed", + ".xls": "application/vnd.ms-excel", + ".xlm": "application/vnd.ms-excel", + ".xla": "application/vnd.ms-excel", + ".xlc": "application/vnd.ms-excel", + ".xlt": "application/vnd.ms-excel", + ".xlw": "application/vnd.ms-excel", + ".xltm": "application/vnd.ms-excel.template.macroEnabled.12", + ".xlam": "application/vnd.ms-excel.addin.macroEnabled.12", + ".xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", + ".eot": "application/vnd.ms-fontobject", + ".chm": "application/vnd.ms-htmlhelp", + ".ims": "application/vnd.ms-ims", + ".lrm": "application/vnd.ms-lrm", + ".thmx": "application/vnd.ms-officetheme", + ".ppt": "application/vnd.ms-powerpoint", + ".pps": "application/vnd.ms-powerpoint", + ".pot": "application/vnd.ms-powerpoint", + ".ppam": "application/vnd.ms-powerpoint.addin.macroEnabled.12", + ".pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + ".sldm": "application/vnd.ms-powerpoint.slide.macroEnabled.12", + ".ppsm": "application/vnd.ms-powerpoint.slideshow.macroEnabled.12", + ".potm": "application/vnd.ms-powerpoint.template.macroEnabled.12", + ".mpp": "application/vnd.ms-project", + ".mpt": "application/vnd.ms-project", + ".tnef": "application/vnd.ms-tnef", + ".tnf": "application/vnd.ms-tnef", + ".docm": "application/vnd.ms-word.document.macroEnabled.12", + ".dotm": "application/vnd.ms-word.template.macroEnabled.12", + ".wcm": "application/vnd.ms-works", + ".wdb": "application/vnd.ms-works", + ".wks": "application/vnd.ms-works", + ".wps": "application/vnd.ms-works", + ".wpl": "application/vnd.ms-wpl", + ".xps": "application/vnd.ms-xpsdocument", + ".msa": "application/vnd.msa-disk-image", + ".mseq": "application/vnd.mseq", + ".crtr": "application/vnd.multiad.creator", + ".cif": "application/vnd.multiad.creator.cif", + ".mus": "application/vnd.musician", + ".msty": "application/vnd.muvee.style", + ".taglet": "application/vnd.mynfc", + ".entity": "application/vnd.nervana", + ".request": "application/vnd.nervana", + ".bkm": "application/vnd.nervana", + ".kcm": "application/vnd.nervana", + ".nitf": "application/vnd.nitf", + ".nlu": "application/vnd.neurolanguage.nlu", + ".nds": "application/vnd.nintendo.nitro.rom", + ".sfc": "application/vnd.nintendo.snes.rom", + ".smc": "application/vnd.nintendo.snes.rom", + ".nnd": "application/vnd.noblenet-directory", + ".nns": "application/vnd.noblenet-sealer", + ".nnw": "application/vnd.noblenet-web", + ".ac": "application/vnd.nokia.n-gage.ac+xml", + ".ngdat": "application/vnd.nokia.n-gage.data", + ".n-gage": "application/vnd.nokia.n-gage.symbian.install", + ".rpst": "application/vnd.nokia.radio-preset", + ".rpss": "application/vnd.nokia.radio-presets", + ".edm": "application/vnd.novadigm.EDM", + ".edx": "application/vnd.novadigm.EDX", + ".ext": "application/vnd.novadigm.EXT", + ".odc": "application/vnd.oasis.opendocument.chart", + ".otc": "application/vnd.oasis.opendocument.chart-template", + ".odb": "application/vnd.oasis.opendocument.database", + ".odf": "application/vnd.oasis.opendocument.formula", + ".odg": "application/vnd.oasis.opendocument.graphics", + ".otg": "application/vnd.oasis.opendocument.graphics-template", + ".odi": "application/vnd.oasis.opendocument.image", + ".oti": "application/vnd.oasis.opendocument.image-template", + ".odp": "application/vnd.oasis.opendocument.presentation", + ".otp": "application/vnd.oasis.opendocument.presentation-template", + ".ods": "application/vnd.oasis.opendocument.spreadsheet", + ".ots": "application/vnd.oasis.opendocument.spreadsheet-template", + ".odt": "application/vnd.oasis.opendocument.text", + ".odm": "application/vnd.oasis.opendocument.text-master", + ".ott": "application/vnd.oasis.opendocument.text-template", + ".oth": "application/vnd.oasis.opendocument.text-web", + ".xo": "application/vnd.olpc-sugar", + ".dd2": "application/vnd.oma.dd2+xml", + ".tam": "application/vnd.onepager", + ".tamp": "application/vnd.onepagertamp", + ".tamx": "application/vnd.onepagertamx", + ".tat": "application/vnd.onepagertat", + ".tatp": "application/vnd.onepagertatp", + ".tatx": "application/vnd.onepagertatx", + ".obgx": "application/vnd.openblox.game+xml", + ".obg": "application/vnd.openblox.game-binary", + ".oeb": "application/vnd.openeye.oeb", + ".oxt": "application/vnd.openofficeorg.extension", + ".osm": "application/vnd.openstreetmap.data+xml", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + ".ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + ".potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + ".ndc": "application/vnd.osa.netdeploy", + ".mgp": "application/vnd.osgeo.mapguide.package", + ".dp": "application/vnd.osgi.dp", + ".esa": "application/vnd.osgi.subsystem", + ".oxlicg": "application/vnd.oxli.countgraph", + ".prc": "application/vnd.palm", + ".pdb": "application/vnd.palm", + ".pqa": "application/vnd.palm", + ".oprc": "application/vnd.palm", + ".plp": "application/vnd.panoply", + ".paw": "application/vnd.pawaafile", + ".str": "application/vnd.pg.format", + ".ei6": "application/vnd.pg.osasli", + ".pil": "application/vnd.piaccess.application-license", + ".efif": "application/vnd.picsel", + ".wg": "application/vnd.pmi.widget", + ".plf": "application/vnd.pocketlearn", + ".pbd": "application/vnd.powerbuilder6", + ".preminet": "application/vnd.preminet", + ".box": "application/vnd.previewsystems.box", + ".vbox": "application/vnd.previewsystems.box", + ".mgz": "application/vnd.proteus.magazine", + ".qps": "application/vnd.publishare-delta-tree", + ".ptid": "application/vnd.pvi.ptid1", + ".bar": "application/vnd.qualcomm.brew-app-res", + ".qxd": "application/vnd.Quark.QuarkXPress", + ".qxt": "application/vnd.Quark.QuarkXPress", + ".qwd": "application/vnd.Quark.QuarkXPress", + ".qwt": "application/vnd.Quark.QuarkXPress", + ".qxl": "application/vnd.Quark.QuarkXPress", + ".qxb": "application/vnd.Quark.QuarkXPress", + ".quox": "application/vnd.quobject-quoxdocument", + ".quiz": "application/vnd.quobject-quoxdocument", + ".tree": "application/vnd.rainstor.data", + ".rar": "application/vnd.rar", + ".bed": "application/vnd.realvnc.bed", + ".mxl": "application/vnd.recordare.musicxml", + ".cryptonote": "application/vnd.rig.cryptonote", + ".link66": "application/vnd.route66.link66+xml", + ".st": "application/vnd.sailingtracker.track", + ".scd": "application/vnd.scribus", + ".sla": "application/vnd.scribus", + ".slaz": "application/vnd.scribus", + ".s3df": "application/vnd.sealed.3df", + ".scsf": "application/vnd.sealed.csf", + ".sdoc": "application/vnd.sealed.doc", + ".sdo": "application/vnd.sealed.doc", + ".s1w": "application/vnd.sealed.doc", + ".seml": "application/vnd.sealed.eml", + ".sem": "application/vnd.sealed.eml", + ".smht": "application/vnd.sealed.mht", + ".smh": "application/vnd.sealed.mht", + ".sppt": "application/vnd.sealed.ppt", + ".s1p": "application/vnd.sealed.ppt", + ".stif": "application/vnd.sealed.tiff", + ".sxls": "application/vnd.sealed.xls", + ".sxl": "application/vnd.sealed.xls", + ".s1e": "application/vnd.sealed.xls", + ".stml": "application/vnd.sealedmedia.softseal.html", + ".s1h": "application/vnd.sealedmedia.softseal.html", + ".spdf": "application/vnd.sealedmedia.softseal.pdf", + ".spd": "application/vnd.sealedmedia.softseal.pdf", + ".s1a": "application/vnd.sealedmedia.softseal.pdf", + ".see": "application/vnd.seemail", + ".sema": "application/vnd.sema", + ".semd": "application/vnd.semd", + ".semf": "application/vnd.semf", + ".ifm": "application/vnd.shana.informed.formdata", + ".itp": "application/vnd.shana.informed.formtemplate", + ".iif": "application/vnd.shana.informed.interchange", + ".ipk": "application/vnd.shana.informed.package", + ".twd": "application/vnd.SimTech-MindMapper", + ".twds": "application/vnd.SimTech-MindMapper", + ".mmf": "application/vnd.smaf", + ".notebook": "application/vnd.smart.notebook", + ".teacher": "application/vnd.smart.teacher", + ".fo": "application/vnd.software602.filler.form+xml", + ".zfo": "application/vnd.software602.filler.form-xml-zip", + ".sdkm": "application/vnd.solent.sdkm+xml", + ".sdkd": "application/vnd.solent.sdkm+xml", + ".dxp": "application/vnd.spotfire.dxp", + ".sfs": "application/vnd.spotfire.sfs", + ".smzip": "application/vnd.stepmania.package", + ".sm": "application/vnd.stepmania.stepchart", + ".wadl": "application/vnd.sun.wadl+xml", + ".sus": "application/vnd.sus-calendar", + ".susp": "application/vnd.sus-calendar", + ".xsm": "application/vnd.syncml+xml", + ".bdm": "application/vnd.syncml.dm+wbxml", + ".xdm": "application/vnd.syncml.dm+xml", + ".ddf": "application/vnd.syncml.dmddf+xml", + ".tao": "application/vnd.tao.intent-module-archive", + ".pcap": "application/vnd.tcpdump.pcap", + ".cap": "application/vnd.tcpdump.pcap", + ".dmp": "application/vnd.tcpdump.pcap", + ".qvd": "application/vnd.theqvd", + ".vfr": "application/vnd.tml", + ".viaframe": "application/vnd.tml", + ".tmo": "application/vnd.tmobile-livetv", + ".tpt": "application/vnd.trid.tpt", + ".mxs": "application/vnd.triscape.mxs", + ".tra": "application/vnd.trueapp", + ".ufdl": "application/vnd.ufdl", + ".ufd": "application/vnd.ufdl", + ".frm": "application/vnd.ufdl", + ".utz": "application/vnd.uiq.theme", + ".umj": "application/vnd.umajin", + ".unityweb": "application/vnd.unity", + ".uoml": "application/vnd.uoml+xml", + ".uo": "application/vnd.uoml+xml", + ".urim": "application/vnd.uri-map", + ".urimap": "application/vnd.uri-map", + ".vmt": "application/vnd.valve.source.material", + ".vcx": "application/vnd.vcx", + ".mxi": "application/vnd.vd-study", + ".study-inter": "application/vnd.vd-study", + ".model-inter": "application/vnd.vd-study", + ".vwx": "application/vnd.vectorworks", + ".vsc": "application/vnd.vidsoft.vidconference", + ".vsd": "application/vnd.visio", + ".vst": "application/vnd.visio", + ".vsw": "application/vnd.visio", + ".vss": "application/vnd.visio", + ".vis": "application/vnd.visionary", + ".vsf": "application/vnd.vsf", + ".sic": "application/vnd.wap.sic", + ".slc": "application/vnd.wap.slc", + ".wbxml": "application/vnd.wap.wbxml", + ".wmlc": "application/vnd.wap.wmlc", + ".wmlsc": "application/vnd.wap.wmlscriptc", + ".wtb": "application/vnd.webturbo", + ".p2p": "application/vnd.wfa.p2p", + ".wsc": "application/vnd.wfa.wsc", + ".wmc": "application/vnd.wmc", + ".m": "application/vnd.wolfram.mathematica.package", + ".nbp": "application/vnd.wolfram.player", + ".wpd": "application/vnd.wordperfect", + ".wqd": "application/vnd.wqd", + ".stf": "application/vnd.wt.stf", + ".wv": "application/vnd.wv.csp+wbxml", + ".xar": "application/vnd.xara", + ".xfdl": "application/vnd.xfdl", + ".xfd": "application/vnd.xfdl", + ".cpkg": "application/vnd.xmpie.cpkg", + ".dpkg": "application/vnd.xmpie.dpkg", + ".ppkg": "application/vnd.xmpie.ppkg", + ".xlim": "application/vnd.xmpie.xlim", + ".hvd": "application/vnd.yamaha.hv-dic", + ".hvs": "application/vnd.yamaha.hv-script", + ".hvp": "application/vnd.yamaha.hv-voice", + ".osf": "application/vnd.yamaha.openscoreformat", + ".saf": "application/vnd.yamaha.smaf-audio", + ".spf": "application/vnd.yamaha.smaf-phrase", + ".yme": "application/vnd.yaoweme", + ".cmp": "application/vnd.yellowriver-custom-menu", + ".zir": "application/vnd.zul", + ".zirz": "application/vnd.zul", + ".zaz": "application/vnd.zzazz.deck+xml", + ".vxml": "application/voicexml+xml", + ".wif": "application/watcherinfo+xml", + ".wgt": "application/widget", + ".wsdl": "application/wsdl+xml", + ".wspolicy": "application/wspolicy+xml", + ".xav": "application/xcap-att+xml", + ".xca": "application/xcap-caps+xml", + ".xdf": "application/xcap-diff+xml", + ".xel": "application/xcap-el+xml", + ".xer": "application/xcap-error+xml", + ".xns": "application/xcap-ns+xml", + ".xhtml": "application/xhtml+xml", + ".xhtm": "application/xhtml+xml", + ".xht": "application/xhtml+xml", + ".dtd": "application/xml-dtd", + ".xop": "application/xop+xml", + ".xsl": "application/xslt+xml", + ".xslt": "application/xslt+xml", + ".mxml": "application/xv+xml", + ".xhvml": "application/xv+xml", + ".xvml": "application/xv+xml", + ".xvm": "application/xv+xml", + ".yang": "application/yang", + ".yin": "application/yin+xml", + ".zip": "application/zip", + ".726": "audio/32kadpcm", + ".ac3": "audio/ac3", + ".amr": "audio/AMR", + ".awb": "audio/AMR-WB", + ".acn": "audio/asc", + ".aal": "audio/ATRAC-ADVANCED-LOSSLESS", + ".atx": "audio/ATRAC-X", + ".at3": "audio/ATRAC3", + ".aa3": "audio/ATRAC3", + ".omg": "audio/ATRAC3", + ".au": "audio/basic", + ".snd": "audio/basic", + ".dls": "audio/dls", + ".evc": "audio/EVRC", + ".evb": "audio/EVRCB", + ".enw": "audio/EVRCNW", + ".evw": "audio/EVRCWB", + ".lbc": "audio/iLBC", + ".l16": "audio/L16", + ".mxmf": "audio/mobile-xmf", + ".m4a": "audio/mp4", + ".mp3": "audio/mpeg", + ".mpga": "audio/mpeg", + ".mp1": "audio/mpeg", + ".mp2": "audio/mpeg", + ".oga": "audio/ogg", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + ".spx": "audio/ogg", + ".sid": "audio/prs.sid", + ".psid": "audio/prs.sid", + ".qcp": "audio/qcelp", + ".smv": "audio/SMV", + ".koz": "audio/vnd.audikoz", + ".uva": "audio/vnd.dece.audio", + ".uvva": "audio/vnd.dece.audio", + ".eol": "audio/vnd.digital-winds", + ".mlp": "audio/vnd.dolby.mlp", + ".dts": "audio/vnd.dts", + ".dtshd": "audio/vnd.dts.hd", + ".plj": "audio/vnd.everad.plj", + ".lvp": "audio/vnd.lucent.voice", + ".pya": "audio/vnd.ms-playready.media.pya", + ".vbk": "audio/vnd.nortel.vbk", + ".ecelp4800": "audio/vnd.nuera.ecelp4800", + ".ecelp7470": "audio/vnd.nuera.ecelp7470", + ".ecelp9600": "audio/vnd.nuera.ecelp9600", + ".rip": "audio/vnd.rip", + ".smp3": "audio/vnd.sealedmedia.softseal.mpeg", + ".smp": "audio/vnd.sealedmedia.softseal.mpeg", + ".s1m": "audio/vnd.sealedmedia.softseal.mpeg", + ".ttc": "font/collection", + ".otf": "font/otf", + ".ttf": "font/ttf", + ".woff": "font/woff", + ".woff2": "font/woff2", + ".bmp": "image/bmp", + ".dib": "image/bmp", + ".cgm": "image/cgm", + ".drle": "image/dicom-rle", + ".emf": "image/emf", + ".fits": "image/fits", + ".fit": "image/fits", + ".fts": "image/fits", + ".gif": "image/gif", + ".ief": "image/ief", + ".jls": "image/jls", + ".jp2": "image/jp2", + ".jpg2": "image/jp2", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".jpe": "image/jpeg", + ".jfif": "image/jpeg", + ".jpm": "image/jpm", + ".jpgm": "image/jpm", + ".jpx": "image/jpx", + ".jpf": "image/jpx", + ".ktx": "image/ktx", + ".png": "image/png", + ".btif": "image/prs.btif", + ".btf": "image/prs.btif", + ".pti": "image/prs.pti", + ".svg": "image/svg+xml", + ".svgz": "image/svg+xml", + ".t38": "image/t38", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".tfx": "image/tiff-fx", + ".psd": "image/vnd.adobe.photoshop", + ".azv": "image/vnd.airzip.accelerator.azv", + ".uvi": "image/vnd.dece.graphic", + ".uvvi": "image/vnd.dece.graphic", + ".uvg": "image/vnd.dece.graphic", + ".uvvg": "image/vnd.dece.graphic", + ".djvu": "image/vnd.djvu", + ".djv": "image/vnd.djvu", + ".dwg": "image/vnd.dwg", + ".dxf": "image/vnd.dxf", + ".fbs": "image/vnd.fastbidsheet", + ".fpx": "image/vnd.fpx", + ".fst": "image/vnd.fst", + ".mmr": "image/vnd.fujixerox.edmics-mmr", + ".rlc": "image/vnd.fujixerox.edmics-rlc", + ".pgb": "image/vnd.globalgraphics.pgb", + ".ico": "image/vnd.microsoft.icon", + ".apng": "image/vnd.mozilla.apng", + ".mdi": "image/vnd.ms-modi", + ".hdr": "image/vnd.radiance", + ".rgbe": "image/vnd.radiance", + ".xyze": "image/vnd.radiance", + ".spng": "image/vnd.sealed.png", + ".spn": "image/vnd.sealed.png", + ".s1n": "image/vnd.sealed.png", + ".sgif": "image/vnd.sealedmedia.softseal.gif", + ".sgi": "image/vnd.sealedmedia.softseal.gif", + ".s1g": "image/vnd.sealedmedia.softseal.gif", + ".sjpg": "image/vnd.sealedmedia.softseal.jpg", + ".sjp": "image/vnd.sealedmedia.softseal.jpg", + ".s1j": "image/vnd.sealedmedia.softseal.jpg", + ".tap": "image/vnd.tencent.tap", + ".vtf": "image/vnd.valve.source.texture", + ".wbmp": "image/vnd.wap.wbmp", + ".xif": "image/vnd.xiff", + ".pcx": "image/vnd.zbrush.pcx", + ".wmf": "image/wmf", + ".u8msg": "message/global", + ".u8dsn": "message/global-delivery-status", + ".u8mdn": "message/global-disposition-notification", + ".u8hdr": "message/global-headers", + ".eml": "message/rfc822", + ".mail": "message/rfc822", + ".art": "message/rfc822", + ".gltf": "model/gltf+json", + ".igs": "model/iges", + ".iges": "model/iges", + ".msh": "model/mesh", + ".mesh": "model/mesh", + ".silo": "model/mesh", + ".dae": "model/vnd.collada+xml", + ".dwf": "model/vnd.dwf", + ".gdl": "model/vnd.gdl", + ".gsm": "model/vnd.gdl", + ".win": "model/vnd.gdl", + ".dor": "model/vnd.gdl", + ".lmp": "model/vnd.gdl", + ".rsm": "model/vnd.gdl", + ".msm": "model/vnd.gdl", + ".ism": "model/vnd.gdl", + ".gtw": "model/vnd.gtw", + ".moml": "model/vnd.moml+xml", + ".mts": "model/vnd.mts", + ".ogex": "model/vnd.opengex", + ".x_b": "model/vnd.parasolid.transmit.binary", + ".xmt_bin": "model/vnd.parasolid.transmit.binary", + ".x_t": "model/vnd.parasolid.transmit.text", + ".xmt_txt": "model/vnd.parasolid.transmit.text", + ".bsp": "model/vnd.valve.source.compiled-map", + ".vtu": "model/vnd.vtu", + ".wrl": "model/vrml", + ".vrml": "model/vrml", + ".x3db": "model/x3d+xml", + ".x3dv": "model/x3d-vrml", + ".x3dvz": "model/x3d-vrml", + ".bmed": "multipart/vnd.bint.med-plus", + ".vpm": "multipart/voice-message", + ".appcache": "text/cache-manifest", + ".manifest": "text/cache-manifest", + ".ics": "text/calendar", + ".ifb": "text/calendar", + ".css": "text/css", + ".csv": "text/csv", + ".csvs": "text/csv-schema", + ".soa": "text/dns", + ".zone": "text/dns", + ".html": "text/html", + ".htm": "text/html", + ".cnd": "text/jcr-cnd", + ".markdown": "text/markdown", + ".md": "text/markdown", + ".miz": "text/mizar", + ".n3": "text/n3", + ".txt": "text/plain", + ".asc": "text/plain", + ".text": "text/plain", + ".pm": "text/plain", + ".el": "text/plain", + ".c": "text/plain", + ".h": "text/plain", + ".cc": "text/plain", + ".hh": "text/plain", + ".cxx": "text/plain", + ".hxx": "text/plain", + ".f90": "text/plain", + ".conf": "text/plain", + ".log": "text/plain", + ".provn": "text/provenance-notation", + ".rst": "text/prs.fallenstein.rst", + ".tag": "text/prs.lines.tag", + ".dsc": "text/prs.lines.tag", + ".rtx": "text/richtext", + ".sgml": "text/sgml", + ".sgm": "text/sgml", + ".tsv": "text/tab-separated-values", + ".t": "text/troff", + ".tr": "text/troff", + ".roff": "text/troff", + ".ttl": "text/turtle", + ".uris": "text/uri-list", + ".uri": "text/uri-list", + ".vcf": "text/vcard", + ".vcard": "text/vcard", + ".a": "text/vnd.a", + ".abc": "text/vnd.abc", + ".ascii": "text/vnd.ascii-art", + ".copyright": "text/vnd.debian.copyright", + ".dms": "text/vnd.DMClientScript", + ".sub": "text/vnd.dvb.subtitle", + ".jtd": "text/vnd.esmertec.theme-descriptor", + ".fly": "text/vnd.fly", + ".flx": "text/vnd.fmi.flexstor", + ".gv": "text/vnd.graphviz", + ".dot": "text/vnd.graphviz", + ".3dml": "text/vnd.in3d.3dml", + ".3dm": "text/vnd.in3d.3dml", + ".spot": "text/vnd.in3d.spot", + ".spo": "text/vnd.in3d.spot", + ".mpf": "text/vnd.ms-mediapackage", + ".ccc": "text/vnd.net2phone.commcenter.command", + ".uric": "text/vnd.si.uricatalogue", + ".jad": "text/vnd.sun.j2me.app-descriptor", + ".ts": "text/vnd.trolltech.linguist", + ".si": "text/vnd.wap.si", + ".sl": "text/vnd.wap.sl", + ".wml": "text/vnd.wap.wml", + ".wmls": "text/vnd.wap.wmlscript", + ".xml": "text/xml", + ".xsd": "text/xml", + ".rng": "text/xml", + ".ent": "text/xml-external-parsed-entity", + ".3gp": "video/3gpp", + ".3gpp": "video/3gpp", + ".3g2": "video/3gpp2", + ".3gpp2": "video/3gpp2", + ".m4s": "video/iso.segment", + ".mj2": "video/mj2", + ".mjp2": "video/mj2", + ".mp4": "video/mp4", + ".mpg4": "video/mp4", + ".m4v": "video/mp4", + ".mpeg": "video/mpeg", + ".mpg": "video/mpeg", + ".mpe": "video/mpeg", + ".m1v": "video/mpeg", + ".m2v": "video/mpeg", + ".ogv": "video/ogg", + ".mov": "video/quicktime", + ".qt": "video/quicktime", + ".uvh": "video/vnd.dece.hd", + ".uvvh": "video/vnd.dece.hd", + ".uvm": "video/vnd.dece.mobile", + ".uvvm": "video/vnd.dece.mobile", + ".uvu": "video/vnd.dece.mp4", + ".uvvu": "video/vnd.dece.mp4", + ".uvp": "video/vnd.dece.pd", + ".uvvp": "video/vnd.dece.pd", + ".uvs": "video/vnd.dece.sd", + ".uvvs": "video/vnd.dece.sd", + ".uvv": "video/vnd.dece.video", + ".uvvv": "video/vnd.dece.video", + ".dvb": "video/vnd.dvb.file", + ".fvt": "video/vnd.fvt", + ".mxu": "video/vnd.mpegurl", + ".m4u": "video/vnd.mpegurl", + ".pyv": "video/vnd.ms-playready.media.pyv", + ".nim": "video/vnd.nokia.interleaved-multimedia", + ".bik": "video/vnd.radgamettools.bink", + ".bk2": "video/vnd.radgamettools.bink", + ".smk": "video/vnd.radgamettools.smacker", + ".smpg": "video/vnd.sealed.mpeg1", + ".s11": "video/vnd.sealed.mpeg1", + ".s14": "video/vnd.sealed.mpeg4", + ".sswf": "video/vnd.sealed.swf", + ".ssw": "video/vnd.sealed.swf", + ".smov": "video/vnd.sealedmedia.softseal.mov", + ".smo": "video/vnd.sealedmedia.softseal.mov", + ".s1q": "video/vnd.sealedmedia.softseal.mov", + ".viv": "video/vnd.vivo", + ".cpt": "application/mac-compactpro", + ".metalink": "application/metalink+xml", + ".owx": "application/owl+xml", + ".rss": "application/rss+xml", + ".apk": "application/vnd.android.package-archive", + ".dd": "application/vnd.oma.dd+xml", + ".dcf": "application/vnd.oma.drm.content", + ".o4a": "application/vnd.oma.drm.dcf", + ".o4v": "application/vnd.oma.drm.dcf", + ".dm": "application/vnd.oma.drm.message", + ".drc": "application/vnd.oma.drm.rights+wbxml", + ".dr": "application/vnd.oma.drm.rights+xml", + ".sxc": "application/vnd.sun.xml.calc", + ".stc": "application/vnd.sun.xml.calc.template", + ".sxd": "application/vnd.sun.xml.draw", + ".std": "application/vnd.sun.xml.draw.template", + ".sxi": "application/vnd.sun.xml.impress", + ".sti": "application/vnd.sun.xml.impress.template", + ".sxm": "application/vnd.sun.xml.math", + ".sxw": "application/vnd.sun.xml.writer", + ".sxg": "application/vnd.sun.xml.writer.global", + ".stw": "application/vnd.sun.xml.writer.template", + ".sis": "application/vnd.symbian.install", + ".mms": "application/vnd.wap.mms-message", + ".anx": "application/x-annodex", + ".bcpio": "application/x-bcpio", + ".torrent": "application/x-bittorrent", + ".bz2": "application/x-bzip2", + ".vcd": "application/x-cdlink", + ".crx": "application/x-chrome-extension", + ".cpio": "application/x-cpio", + ".csh": "application/x-csh", + ".dcr": "application/x-director", + ".dir": "application/x-director", + ".dxr": "application/x-director", + ".dvi": "application/x-dvi", + ".spl": "application/x-futuresplash", + ".gtar": "application/x-gtar", + ".hdf": "application/x-hdf", + ".jar": "application/x-java-archive", + ".jnlp": "application/x-java-jnlp-file", + ".pack": "application/x-java-pack200", + ".kil": "application/x-killustrator", + ".latex": "application/x-latex", + ".nc": "application/x-netcdf", + ".cdf": "application/x-netcdf", + ".pl": "application/x-perl", + ".rpm": "application/x-rpm", + ".sh": "application/x-sh", + ".shar": "application/x-shar", + ".sit": "application/x-stuffit", + ".sv4cpio": "application/x-sv4cpio", + ".sv4crc": "application/x-sv4crc", + ".tar": "application/x-tar", + ".tcl": "application/x-tcl", + ".tex": "application/x-tex", + ".texinfo": "application/x-texinfo", + ".texi": "application/x-texinfo", + ".man": "application/x-troff-man", + ".1": "application/x-troff-man", + ".2": "application/x-troff-man", + ".3": "application/x-troff-man", + ".4": "application/x-troff-man", + ".5": "application/x-troff-man", + ".6": "application/x-troff-man", + ".7": "application/x-troff-man", + ".8": "application/x-troff-man", + ".me": "application/x-troff-me", + ".ms": "application/x-troff-ms", + ".ustar": "application/x-ustar", + ".src": "application/x-wais-source", + ".xpi": "application/x-xpinstall", + ".xspf": "application/x-xspf+xml", + ".xz": "application/x-xz", + ".mid": "audio/midi", + ".midi": "audio/midi", + ".kar": "audio/midi", + ".aif": "audio/x-aiff", + ".aiff": "audio/x-aiff", + ".aifc": "audio/x-aiff", + ".axa": "audio/x-annodex", + ".flac": "audio/x-flac", + ".mka": "audio/x-matroska", + ".mod": "audio/x-mod", + ".ult": "audio/x-mod", + ".uni": "audio/x-mod", + ".m15": "audio/x-mod", + ".mtm": "audio/x-mod", + ".669": "audio/x-mod", + ".med": "audio/x-mod", + ".m3u": "audio/x-mpegurl", + ".wax": "audio/x-ms-wax", + ".wma": "audio/x-ms-wma", + ".ram": "audio/x-pn-realaudio", + ".rm": "audio/x-pn-realaudio", + ".ra": "audio/x-realaudio", + ".s3m": "audio/x-s3m", + ".stm": "audio/x-stm", + ".wav": "audio/x-wav", + ".xyz": "chemical/x-xyz", + ".webp": "image/webp", + ".ras": "image/x-cmu-raster", + ".pnm": "image/x-portable-anymap", + ".pbm": "image/x-portable-bitmap", + ".pgm": "image/x-portable-graymap", + ".ppm": "image/x-portable-pixmap", + ".rgb": "image/x-rgb", + ".tga": "image/x-targa", + ".xbm": "image/x-xbitmap", + ".xpm": "image/x-xpixmap", + ".xwd": "image/x-xwindowdump", + ".sandboxed": "text/html-sandboxed", + ".pod": "text/x-pod", + ".etx": "text/x-setext", + ".webm": "video/webm", + ".axv": "video/x-annodex", + ".flv": "video/x-flv", + ".fxm": "video/x-javafx", + ".mkv": "video/x-matroska", + ".mk3d": "video/x-matroska-3d", + ".asx": "video/x-ms-asf", + ".wm": "video/x-ms-wm", + ".wmv": "video/x-ms-wmv", + ".wmx": "video/x-ms-wmx", + ".wvx": "video/x-ms-wvx", + ".avi": "video/x-msvideo", + ".movie": "video/x-sgi-movie", + ".ice": "x-conference/x-cooltalk", + ".sisx": "x-epoc/x-sisx-app", + } + for ext, name := range mimeTypes { + if err := mime.AddExtensionType(ext, name); err != nil { + panic(err) + } + } +} diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 3b2dcc7d59..f5f70138b3 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -50,7 +50,7 @@ func ParseURI(h http.Handler) http.Handler { uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) if err != nil { w.WriteHeader(http.StatusBadRequest) - RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) + respondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) return } if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") { @@ -75,7 +75,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) - log.Debug("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) }) } diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index c9fb9d2855..d4e81d7f67 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -53,23 +53,23 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) msg := "" if list.Entries == nil { - RespondError(w, r, "Could not resolve", http.StatusInternalServerError) + respondError(w, r, "Could not resolve", http.StatusInternalServerError) return } requestUri := strings.TrimPrefix(r.RequestURI, "/") uri, err := api.Parse(requestUri) if err != nil { - RespondError(w, r, "Bad Request", http.StatusBadRequest) + respondError(w, r, "Bad Request", http.StatusBadRequest) } uri.Scheme = "bzz-list" msg += fmt.Sprintf("Disambiguation:
Your request may refer to multiple choices.
Click here if your browser does not redirect you within 5 seconds.
", "/"+uri.String()) - RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices) + respondTemplate(w, r, "error", msg, http.StatusMultipleChoices) } -func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { - log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) +func respondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) { + log.Debug("respondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) respond(w, r, &ResponseParams{ Code: code, Msg: template.HTML(msg), @@ -78,13 +78,12 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s }) } -func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) { - log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) - RespondTemplate(w, r, "error", msg, code) +func respondError(w http.ResponseWriter, r *http.Request, msg string, code int) { + log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) + respondTemplate(w, r, "error", msg, code) } func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { - w.WriteHeader(params.Code) if params.Code >= 400 { @@ -96,7 +95,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { // this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8" if strings.Contains(acceptHeader, "application/json") { if err := respondJSON(w, r, params); err != nil { - RespondError(w, r, "Internal server error", http.StatusInternalServerError) + respondError(w, r, "Internal server error", http.StatusInternalServerError) } } else if strings.Contains(acceptHeader, "text/html") { respondHTML(w, r, params) @@ -107,7 +106,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) { htmlCounter.Inc(1) - log.Debug("respondHTML", "ruid", GetRUID(r.Context())) + log.Info("respondHTML", "ruid", GetRUID(r.Context()), "code", params.Code) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -116,14 +115,14 @@ func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { jsonCounter.Inc(1) - log.Debug("respondJSON", "ruid", GetRUID(r.Context())) + log.Info("respondJSON", "ruid", GetRUID(r.Context()), "code", params.Code) w.Header().Set("Content-Type", "application/json") return json.NewEncoder(w).Encode(params) } func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { plaintextCounter.Inc(1) - log.Debug("respondPlaintext", "ruid", GetRUID(r.Context())) + log.Info("respondPlaintext", "ruid", GetRUID(r.Context()), "code", params.Code) w.Header().Set("Content-Type", "text/plain") strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n" strToWrite += "Message: " + string(params.Msg) + "\n" diff --git a/swarm/api/http/response_test.go b/swarm/api/http/response_test.go index 50a704be6d..486c19ab0e 100644 --- a/swarm/api/http/response_test.go +++ b/swarm/api/http/response_test.go @@ -24,12 +24,10 @@ import ( "testing" "golang.org/x/net/html" - - "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestError(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -55,7 +53,7 @@ func TestError(t *testing.T) { } func Test404Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -81,7 +79,7 @@ func Test404Page(t *testing.T) { } func Test500Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -106,7 +104,7 @@ func Test500Page(t *testing.T) { } } func Test500PageWith0xHashPrefix(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -136,7 +134,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) { } func TestJsonResponse(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response diff --git a/swarm/api/http/sctx.go b/swarm/api/http/sctx.go index 431e117354..b8dafab0b7 100644 --- a/swarm/api/http/sctx.go +++ b/swarm/api/http/sctx.go @@ -7,14 +7,10 @@ import ( "github.com/ethereum/go-ethereum/swarm/sctx" ) -type contextKey int - -const ( - uriKey contextKey = iota -) +type uriKey struct{} func GetRUID(ctx context.Context) string { - v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string) + v, ok := ctx.Value(sctx.HTTPRequestIDKey{}).(string) if ok { return v } @@ -22,11 +18,11 @@ func GetRUID(ctx context.Context) string { } func SetRUID(ctx context.Context, ruid string) context.Context { - return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid) + return context.WithValue(ctx, sctx.HTTPRequestIDKey{}, ruid) } func GetURI(ctx context.Context) *api.URI { - v, ok := ctx.Value(uriKey).(*api.URI) + v, ok := ctx.Value(uriKey{}).(*api.URI) if ok { return v } @@ -34,5 +30,5 @@ func GetURI(ctx context.Context) *api.URI { } func SetURI(ctx context.Context, uri *api.URI) context.Context { - return context.WithValue(ctx, uriKey, uri) + return context.WithValue(ctx, uriKey{}, uri) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index af1269b934..3c6735a73e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -31,7 +31,6 @@ import ( "net/http" "os" "path" - "regexp" "strconv" "strings" "time" @@ -41,17 +40,10 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" - + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/rs/cors" ) -type resourceResponse struct { - Manifest storage.Address `json:"manifest"` - Resource string `json:"resource"` - Update storage.Address `json:"update"` -} - var ( postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil) postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil) @@ -145,13 +137,13 @@ func NewServer(api *api.API, corsString string) *Server { defaultMiddlewares..., ), }) - mux.Handle("/bzz-resource:/", methodHandler{ + mux.Handle("/bzz-feed:/", methodHandler{ "GET": Adapt( - http.HandlerFunc(server.HandleGetResource), + http.HandlerFunc(server.HandleGetFeed), defaultMiddlewares..., ), "POST": Adapt( - http.HandlerFunc(server.HandlePostResource), + http.HandlerFunc(server.HandlePostFeed), defaultMiddlewares..., ), }) @@ -192,15 +184,22 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { if err != nil { if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String())) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) return } defer reader.Close() w.Header().Set("Content-Type", "application/x-tar") + + fileName := uri.Addr + if found := path.Base(uri.Path); found != "" && found != "." && found != "/" { + fileName = found + } + w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s.tar\"", fileName)) + w.WriteHeader(http.StatusOK) io.Copy(w, reader) return @@ -212,7 +211,7 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { switch r.RequestURI { case "/": - RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) + respondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200) return case "/robots.txt": w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) @@ -221,7 +220,7 @@ func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(faviconBytes) default: - RespondError(w, r, "Not Found", http.StatusNotFound) + respondError(w, r, "Not Found", http.StatusNotFound) } } @@ -241,26 +240,26 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) { if uri.Path != "" { postRawFail.Inc(1) - RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) + respondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) return } if uri.Addr != "" && uri.Addr != "encrypt" { postRawFail.Inc(1) - RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) + respondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { postRawFail.Inc(1) - RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) + respondError(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -284,7 +283,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusBadRequest) + respondError(w, r, err.Error(), http.StatusBadRequest) return } @@ -299,7 +298,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { addr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) return } log.Debug("resolved key", "ruid", ruid, "key", addr) @@ -307,7 +306,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { addr, err = s.api.NewManifest(r.Context(), toEncrypt) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } log.Debug("new manifest", "ruid", ruid, "key", addr) @@ -318,7 +317,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { case "application/x-tar": _, err := s.handleTarUpload(r, mw) if err != nil { - RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) return err } return nil @@ -331,7 +330,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { }) if err != nil { postFilesFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } @@ -367,7 +366,7 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api } var size int64 - var reader io.Reader = part + var reader io.Reader if contentLength := part.Header.Get("Content-Length"); contentLength != "" { size, err = strconv.ParseInt(contentLength, 10, 64) if err != nil { @@ -403,7 +402,6 @@ func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api Path: path, ContentType: part.Header.Get("Content-Type"), Size: size, - ModTime: time.Now(), } log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path) contentKey, err := mw.AddEntry(r.Context(), reader, entry) @@ -422,7 +420,6 @@ func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) err ContentType: r.Header.Get("Content-Type"), Mode: 0644, Size: r.ContentLength, - ModTime: time.Now(), }) if err != nil { return err @@ -442,7 +439,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) { newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path) if err != nil { deleteFail.Inc(1) - RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) return } @@ -451,219 +448,158 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, newKey) } -// Parses a resource update post url to corresponding action -// possible combinations: -// / add multihash update to existing hash -// /raw add raw update to existing hash -// /# create new resource with first update as mulitihash -// /raw/# create new resource with first update raw -func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) { - re, err := regexp.Compile("^(raw)?/?([0-9]+)?$") - if err != nil { - return isRaw, frequency, err - } - m := re.FindAllStringSubmatch(path, 2) - var freqstr = "0" - if len(m) > 0 { - if m[0][1] != "" { - isRaw = true - } - if m[0][2] != "" { - freqstr = m[0][2] - } - } else if len(path) > 0 { - return isRaw, frequency, fmt.Errorf("invalid path") - } - frequency, err = strconv.ParseUint(freqstr, 10, 64) - return isRaw, frequency, err -} - -// Handles creation of new mutable resources and adding updates to existing mutable resources -// There are two types of updates available, "raw" and "multihash." -// If the latter is used, a subsequent bzz:// GET call to the manifest of the resource will return -// the page that the multihash is pointing to, as if it held a normal swarm content manifest -// -// The POST request admits a JSON structure as defined in the mru package: `mru.updateRequestJSON` -// The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content -func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { +// Handles feed manifest creation and feed updates +// The POST request admits a JSON structure as defined in the feeds package: `feed.updateRequestJSON` +// The requests can be to a) create a feed manifest, b) update a feed or c) both a+b: create a feed manifest and publish a first update +func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) - log.Debug("handle.post.resource", "ruid", ruid) + uri := GetURI(r.Context()) + log.Debug("handle.post.feed", "ruid", ruid) var err error - // Creation and update must send mru.updateRequestJSON JSON structure + // Creation and update must send feed.updateRequestJSON JSON structure body, err := ioutil.ReadAll(r.Body) if err != nil { - RespondError(w, r, err.Error(), http.StatusInternalServerError) - return - } - var updateRequest mru.Request - if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON - RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error + respondError(w, r, err.Error(), http.StatusInternalServerError) return } - if updateRequest.IsUpdate() { + fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query()) + if err != nil { // couldn't parse query string or retrieve manifest + getFail.Inc(1) + httpStatus := http.StatusBadRequest + if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { + httpStatus = http.StatusNotFound + } + respondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus) + return + } + + var updateRequest feed.Request + updateRequest.Feed = *fd + query := r.URL.Query() + + if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters + respondError(w, r, err.Error(), http.StatusBadRequest) + return + } + + switch { + case updateRequest.IsUpdate(): // Verify that the signature is intact and that the signer is authorized - // to update this resource - // Check this early, to avoid creating a resource and then not being able to set its first update. + // to update this feed + // Check this early, to avoid creating a feed and then not being able to set its first update. if err = updateRequest.Verify(); err != nil { - RespondError(w, r, err.Error(), http.StatusForbidden) + respondError(w, r, err.Error(), http.StatusForbidden) return } - } - - if updateRequest.IsNew() { - err = s.api.ResourceCreate(r.Context(), &updateRequest) + _, err = s.api.FeedsUpdate(r.Context(), &updateRequest) if err != nil { - code, err2 := s.translateResourceError(w, r, "resource creation fail", err) - RespondError(w, r, err2.Error(), code) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } - } - - if updateRequest.IsUpdate() { - _, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate) + fallthrough + case query.Get("manifest") == "1": + // we create a manifest so we can retrieve feed updates with bzz:// later + // this manifest has a special "feed type" manifest, and saves the + // feed identification used to retrieve feed updates later + m, err := s.api.NewFeedManifest(r.Context(), &updateRequest.Feed) if err != nil { - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError) return } - } - - // at this point both possible operations (create, update or both) were successful - // so in case it was a new resource, then create a manifest and send it over. - - if updateRequest.IsNew() { - // we create a manifest so we can retrieve the resource with bzz:// later - // this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource - // metadata chunk (rootAddr) - m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex()) - if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) - return - } - // the key to the manifest will be passed back to the client - // the client can access the root chunk key directly through its Hash member - // the manifest key should be set as content in the resolver of the ENS name - // \TODO update manifest key automatically in ENS + // the client can access the feed directly through its Feed member + // the manifest key can be set as content in the resolver of the ENS name outdata, err := json.Marshal(m) if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) + respondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) return } fmt.Fprint(w, string(outdata)) + + w.Header().Add("Content-type", "application/json") + default: + respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest) } - w.Header().Add("Content-type", "application/json") } -// Retrieve mutable resource updates: -// bzz-resource:// - get latest update -// bzz-resource:/// - get latest update on period n -// bzz-resource://// - get update version m of period n -// bzz-resource:///meta - get metadata and next version information -// = ens name or hash -// TODO: Enable pass maxPeriod parameter -func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { +// HandleGetFeed retrieves Swarm feeds updates: +// bzz-feed:// - get latest feed update, given a manifest address +// - or - +// specify user + topic (optional), subtopic name (optional) directly, without manifest: +// bzz-feed://?user=0x...&topic=0x...&name=subtopic name +// topic defaults to 0x000... if not specified. +// name defaults to empty string if not specified. +// thus, empty name and topic refers to the user's default feed. +// +// Optional parameters: +// time=xx - get the latest update before time (in epoch seconds) +// hint.time=xx - hint the lookup algorithm looking for updates at around that time +// hint.level=xx - hint the lookup algorithm looking for updates at around this frequency level +// meta=1 - get feed metadata and status information instead of performing a feed query +// NOTE: meta=1 will be deprecated in the near future +func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) - log.Debug("handle.get.resource", "ruid", ruid) + log.Debug("handle.get.feed", "ruid", ruid) var err error - // resolve the content key. - manifestAddr := uri.Address() - if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) - return - } - } else { - w.Header().Set("Cache-Control", "max-age=2147483648") - } - - // get the root chunk rootAddr from the manifest - rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr) - if err != nil { + fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query()) + if err != nil { // couldn't parse query string or retrieve manifest getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound) + httpStatus := http.StatusBadRequest + if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { + httpStatus = http.StatusNotFound + } + respondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus) return } - log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr) - // determine if the query specifies period and version or it is a metadata query - var params []string - if len(uri.Path) > 0 { - if uri.Path == "meta" { - unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound) - return - } - rawResponse, err := unsignedUpdateRequest.MarshalJSON() - if err != nil { - RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError) - return - } - w.Header().Add("Content-type", "application/json") - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, string(rawResponse)) + if r.URL.Query().Get("meta") == "1" { + unsignedUpdateRequest, err := s.api.FeedsNewRequest(r.Context(), fd) + if err != nil { + getFail.Inc(1) + respondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound) return - } - - params = strings.Split(uri.Path, "/") - + rawResponse, err := unsignedUpdateRequest.MarshalJSON() + if err != nil { + respondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError) + return + } + w.Header().Add("Content-type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, string(rawResponse)) + return } - var name string - var data []byte - now := time.Now() - switch len(params) { - case 0: // latest only - name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupLatest(rootAddr)) - case 2: // specific period and version - var version uint64 - var period uint64 - version, err = strconv.ParseUint(params[1], 10, 32) - if err != nil { - break - } - period, err = strconv.ParseUint(params[0], 10, 32) - if err != nil { - break - } - name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupVersion(rootAddr, uint32(period), uint32(version))) - case 1: // last version of specific period - var period uint64 - period, err = strconv.ParseUint(params[0], 10, 32) - if err != nil { - break - } - name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupLatestVersionInPeriod(rootAddr, uint32(period))) - default: // bogus - err = mru.NewError(storage.ErrInvalidValue, "invalid mutable resource request") + lookupParams := &feed.Query{Feed: *fd} + if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version + respondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest) + return } + data, err := s.api.FeedsLookup(r.Context(), lookupParams) + // any error from the switch statement will end up here if err != nil { - code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err) - RespondError(w, r, err2.Error(), code) + code, err2 := s.translateFeedError(w, r, "feed lookup fail", err) + respondError(w, r, err2.Error(), code) return } // All ok, serve the retrieved update - log.Debug("Found update", "name", name, "ruid", ruid) - w.Header().Set("Content-Type", "application/octet-stream") - http.ServeContent(w, r, "", now, bytes.NewReader(data)) + log.Debug("Found update", "feed", fd.Hex(), "ruid", ruid) + w.Header().Set("Content-Type", api.MimeOctetStream) + http.ServeContent(w, r, "", time.Now(), bytes.NewReader(data)) } -func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) { +func (s *Server) translateFeedError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) { code := 0 defaultErr := fmt.Errorf("%s: %v", supErr, err) - rsrcErr, ok := err.(*mru.Error) + rsrcErr, ok := err.(*feed.Error) if !ok && rsrcErr != nil { code = rsrcErr.Code() } @@ -696,7 +632,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { addr, err := s.api.ResolveURI(r.Context(), uri, pass) if err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. @@ -720,7 +656,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { reader, isEncrypted := s.api.Retrieve(r.Context(), addr) if _, err := reader.Size(r.Context(), nil); err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) return } @@ -730,11 +666,9 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { case uri.Raw(): // allow the request to overwrite the content type using a query // parameter - contentType := "application/octet-stream" if typ := r.URL.Query().Get("content_type"); typ != "" { - contentType = typ + w.Header().Set("Content-Type", typ) } - w.Header().Set("Content-Type", contentType) http.ServeContent(w, r, "", time.Now(), reader) case uri.Hash(): w.Header().Set("Content-Type", "text/plain") @@ -762,7 +696,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { addr, err := s.api.Resolve(r.Context(), uri.Addr) if err != nil { getListFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr) @@ -772,10 +706,10 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { getListFail.Inc(1) if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String())) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -823,7 +757,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFileFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) return } } else { @@ -847,17 +781,17 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { if err != nil { if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } switch status { case http.StatusNotFound: getFileNotFound.Inc(1) - RespondError(w, r, err.Error(), http.StatusNotFound) + respondError(w, r, err.Error(), http.StatusNotFound) default: getFileFail.Inc(1) - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) } return } @@ -870,10 +804,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { getFileFail.Inc(1) if isDecryptError(err) { w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) - RespondError(w, r, err.Error(), http.StatusUnauthorized) + respondError(w, r, err.Error(), http.StatusUnauthorized) return } - RespondError(w, r, err.Error(), http.StatusInternalServerError) + respondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -886,12 +820,21 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(r.Context(), nil); err != nil { getFileNotFound.Inc(1) - RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) + respondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound) return } - w.Header().Set("Content-Type", contentType) - http.ServeContent(w, r, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) + if contentType != "" { + w.Header().Set("Content-Type", contentType) + } + + fileName := uri.Addr + if found := path.Base(uri.Path); found != "" && found != "." && found != "/" { + fileName = found + } + w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", fileName)) + + http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) } // The size of buffer used for bufio.Reader on LazyChunkReader passed to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 4a3ca0429a..1ef3deece2 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -20,7 +20,6 @@ import ( "archive/tar" "bytes" "context" - "crypto/rand" "encoding/json" "errors" "flag" @@ -30,12 +29,16 @@ import ( "math/big" "mime/multipart" "net/http" + "net/url" "os" + "path" "strconv" "strings" "testing" "time" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -44,7 +47,7 @@ import ( swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -54,75 +57,34 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func TestResourcePostMode(t *testing.T) { - path := "" - errstr := "resourcePostMode for '%s' should be raw %v frequency %d, was raw %v, frequency %d" - r, f, err := resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if r || f != 0 { - t.Fatalf(errstr, path, false, 0, r, f) - } - - path = "raw" - r, f, err = resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if !r || f != 0 { - t.Fatalf(errstr, path, true, 0, r, f) - } - - path = "13" - r, f, err = resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if r || f == 0 { - t.Fatalf(errstr, path, false, 13, r, f) - } - - path = "raw/13" - r, f, err = resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if !r || f == 0 { - t.Fatalf(errstr, path, true, 13, r, f) - } - - path = "foo/13" - r, f, err = resourcePostMode(path) - if err == nil { - t.Fatal("resourcePostMode for 'foo/13' should fail, returned error nil") - } -} - -func serverFunc(api *api.API) testutil.TestServer { +func serverFunc(api *api.API) TestServer { return NewServer(api, "") } -func newTestSigner() (*mru.GenericSigner, error) { +func newTestSigner() (*feed.GenericSigner, error) { privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") if err != nil { return nil, err } - return mru.NewGenericSigner(privKey), nil + return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash resource types with bzz:// scheme +// test the transparent resolving of multihash-containing feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a resource update +// first upload data, and store the multihash to the resulting manifest in a feed update // retrieving the update with the multihash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestBzzResourceMultihash(t *testing.T) { +func TestBzzFeedMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // add the data our multihash aliased manifest will point to databytes := "bar" - url := fmt.Sprintf("%s/bzz:/", srv.URL) - resp, err := http.Post(url, "text/plain", bytes.NewReader([]byte(databytes))) + testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL) + resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes))) if err != nil { t.Fatal(err) } @@ -140,33 +102,27 @@ func TestBzzResourceMultihash(t *testing.T) { log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) - // our mutable resource "name" - keybytes := "foo.eth" + topic, _ := feed.NewTopic("foo.eth", nil) + updateRequest := feed.NewFirstRequest(topic) - updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{ - Name: keybytes, - Frequency: 13, - StartTime: srv.GetCurrentTime(), - Owner: signer.Address(), - }) - if err != nil { - t.Fatal(err) - } - updateRequest.SetData(mh, true) + updateRequest.SetData(mh) if err := updateRequest.Sign(signer); err != nil { t.Fatal(err) } log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) - body, err := updateRequest.MarshalJSON() + testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } + query := testUrl.Query() + body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted + query.Set("manifest", "1") // indicate we want a manifest back + testUrl.RawQuery = query.Encode() // create the multihash update - url = fmt.Sprintf("%s/bzz-resource:/", srv.URL) - resp, err = http.Post(url, "application/json", bytes.NewReader(body)) + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } @@ -184,14 +140,14 @@ func TestBzzResourceMultihash(t *testing.T) { t.Fatalf("data %s could not be unmarshaled: %v", b, err) } - correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e" + correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" if rsrcResp.Hex() != correctManifestAddrHex { - t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) + t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) } - // get bzz manifest transparent resource resolve - url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) - resp, err = http.Get(url) + // get bzz manifest transparent feed update resolve + testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) + resp, err = http.Get(testBzzUrl) if err != nil { t.Fatal(err) } @@ -208,46 +164,38 @@ func TestBzzResourceMultihash(t *testing.T) { } } -// Test resource updates using the raw update methods -func TestBzzResource(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) +// Test Swarm feeds using the raw update methods +func TestBzzFeed(t *testing.T) { + srv := NewTestSwarmServer(t, serverFunc, nil) signer, _ := newTestSigner() defer srv.Close() - // our mutable resource "name" - keybytes := "foo.eth" - // data of update 1 - databytes := make([]byte, 666) - _, err := rand.Read(databytes) - if err != nil { - t.Fatal(err) - } + update1Data := testutil.RandomBytes(1, 666) + update1Timestamp := srv.CurrentTime + //data for update 2 + update2Data := []byte("foo") - updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{ - Name: keybytes, - Frequency: 13, - StartTime: srv.GetCurrentTime(), - Owner: signer.Address(), - }) - if err != nil { - t.Fatal(err) - } - updateRequest.SetData(databytes, false) + topic, _ := feed.NewTopic("foo.eth", nil) + updateRequest := feed.NewFirstRequest(topic) + updateRequest.SetData(update1Data) if err := updateRequest.Sign(signer); err != nil { t.Fatal(err) } - body, err := updateRequest.MarshalJSON() + // creates feed and sets update 1 + testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } + urlQuery := testUrl.Query() + body := updateRequest.AppendValues(urlQuery) // this adds all query parameters + urlQuery.Set("manifest", "1") // indicate we want a manifest back + testUrl.RawQuery = urlQuery.Encode() - // creates resource and sets update 1 - url := fmt.Sprintf("%s/bzz-resource:/", srv.URL) - resp, err := http.Post(url, "application/json", bytes.NewReader(body)) + resp, err := http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } @@ -265,14 +213,14 @@ func TestBzzResource(t *testing.T) { t.Fatalf("data %s could not be unmarshaled: %v", b, err) } - correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e" + correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" if rsrcResp.Hex() != correctManifestAddrHex { - t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) } // get the manifest - url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp) - resp, err = http.Get(url) + testRawUrl := fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp) + resp, err = http.Get(testRawUrl) if err != nil { t.Fatal(err) } @@ -292,43 +240,43 @@ func TestBzzResource(t *testing.T) { if len(manifest.Entries) != 1 { t.Fatalf("Manifest has %d entries", len(manifest.Entries)) } - correctRootKeyHex := "68f7ba07ac8867a4c841a4d4320e3cdc549df23702dc7285fcb6acf65df48562" - if manifest.Entries[0].Hash != correctRootKeyHex { - t.Fatalf("Expected manifest path '%s', got '%s'", correctRootKeyHex, manifest.Entries[0].Hash) + correctFeedHex := "0x666f6f2e65746800000000000000000000000000000000000000000000000000c96aaa54e2d44c299564da76e1cd3184a2386b8d" + if manifest.Entries[0].Feed.Hex() != correctFeedHex { + t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex()) } - // get bzz manifest transparent resource resolve - url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) - resp, err = http.Get(url) + // get bzz manifest transparent feed update resolve + testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) + resp, err = http.Get(testBzzUrl) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) + if resp.StatusCode == http.StatusOK { + t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK") } - b, err = ioutil.ReadAll(resp.Body) + _, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } // get non-existent name, should fail - url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL) - resp, err = http.Get(url) + testBzzResUrl := fmt.Sprintf("%s/bzz-feed:/bar", srv.URL) + resp, err = http.Get(testBzzResUrl) if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusNotFound { - t.Fatalf("Expected get non-existent resource to fail with StatusNotFound (404), got %d", resp.StatusCode) + t.Fatalf("Expected get non-existent feed manifest to fail with StatusNotFound (404), got %d", resp.StatusCode) } resp.Body.Close() - // get latest update (1.1) through resource directly + // get latest update through bzz-feed directly log.Info("get update latest = 1.1", "addr", correctManifestAddrHex) - url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex) - resp, err = http.Get(url) + testBzzResUrl = fmt.Sprintf("%s/bzz-feed:/%s", srv.URL, correctManifestAddrHex) + resp, err = http.Get(testBzzResUrl) if err != nil { t.Fatal(err) } @@ -340,54 +288,88 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Equal(databytes, b) { - t.Fatalf("Expected body '%x', got '%x'", databytes, b) + if !bytes.Equal(update1Data, b) { + t.Fatalf("Expected body '%x', got '%x'", update1Data, b) } // update 2 + // Move the clock ahead 1 second + srv.CurrentTime++ log.Info("update 2") - // 1.- get metadata about this resource - url = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex) - resp, err = http.Get(url + "meta") + // 1.- get metadata about this feed + testBzzResUrl = fmt.Sprintf("%s/bzz-feed:/%s/", srv.URL, correctManifestAddrHex) + resp, err = http.Get(testBzzResUrl + "?meta=1") if err != nil { t.Fatal(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - t.Fatalf("Get resource metadata returned %s", resp.Status) + t.Fatalf("Get feed metadata returned %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - updateRequest = &mru.Request{} + updateRequest = &feed.Request{} if err = updateRequest.UnmarshalJSON(b); err != nil { - t.Fatalf("Error decoding resource metadata: %s", err) + t.Fatalf("Error decoding feed metadata: %s", err) } - data := []byte("foo") - updateRequest.SetData(data, false) + updateRequest.SetData(update2Data) if err = updateRequest.Sign(signer); err != nil { t.Fatal(err) } - body, err = updateRequest.MarshalJSON() + testUrl, err = url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } + urlQuery = testUrl.Query() + body = updateRequest.AppendValues(urlQuery) // this adds all query parameters + goodQueryParameters := urlQuery.Encode() // save the query parameters for a second attempt - resp, err = http.Post(url, "application/json", bytes.NewReader(body)) + // create bad query parameters in which the signature is missing + urlQuery.Del("signature") + testUrl.RawQuery = urlQuery.Encode() + + // 1st attempt with bad query parameters in which the signature is missing + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("Update returned %s", resp.Status) + expectedCode := http.StatusBadRequest + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) } - // get latest update (1.2) through resource directly + // 2nd attempt with bad query parameters in which the signature is of incorrect length + urlQuery.Set("signature", "0xabcd") // should be 130 hex chars + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + expectedCode = http.StatusBadRequest + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) + } + + // 3rd attempt, with good query parameters: + testUrl.RawQuery = goodQueryParameters + resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + expectedCode = http.StatusOK + if resp.StatusCode != expectedCode { + t.Fatalf("Update returned %s. Expected %d", resp.Status, expectedCode) + } + + // get latest update through bzz-feed directly log.Info("get update 1.2") - url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex) - resp, err = http.Get(url) + testBzzResUrl = fmt.Sprintf("%s/bzz-feed:/%s", srv.URL, correctManifestAddrHex) + resp, err = http.Get(testBzzResUrl) if err != nil { t.Fatal(err) } @@ -399,33 +381,23 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Equal(data, b) { - t.Fatalf("Expected body '%x', got '%x'", data, b) + if !bytes.Equal(update2Data, b) { + t.Fatalf("Expected body '%x', got '%x'", update2Data, b) } - // get latest update (1.2) with specified period - log.Info("get update latest = 1.2") - url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, correctManifestAddrHex) - resp, err = http.Get(url) + // test manifest-less queries + log.Info("get first update in update1Timestamp via direct query") + query := feed.NewQuery(&updateRequest.Feed, update1Timestamp, lookup.NoClue) + + urlq, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) - } - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, b) { - t.Fatalf("Expected body '%x', got '%x'", data, b) - } - // get first update (1.1) with specified period and version - log.Info("get first update 1.1") - url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, correctManifestAddrHex) - resp, err = http.Get(url) + values := urlq.Query() + query.AppendValues(values) // this adds feed query parameters + urlq.RawQuery = values.Encode() + resp, err = http.Get(urlq.String()) if err != nil { t.Fatal(err) } @@ -437,9 +409,10 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Equal(databytes, b) { - t.Fatalf("Expected body '%x', got '%x'", databytes, b) + if !bytes.Equal(update1Data, b) { + t.Fatalf("Expected body '%x', got '%x'", update1Data, b) } + } func TestBzzGetPath(t *testing.T) { @@ -469,7 +442,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { addr := [3]storage.Address{} - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for i, mf := range testmanifest { @@ -510,6 +483,9 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } defer resp.Body.Close() respbody, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Error while reading response body: %v", err) + } if string(respbody) != testmanifest[v] { isexpectedfailrequest := false @@ -704,7 +680,7 @@ func TestBzzTar(t *testing.T) { } func testBzzTar(encrypted bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} @@ -773,6 +749,16 @@ func testBzzTar(encrypted bool, t *testing.T) { } defer resp2.Body.Close() + if h := resp2.Header.Get("Content-Type"); h != "application/x-tar" { + t.Fatalf("Content-Type header expected: application/x-tar, got: %s", h) + } + + expectedFileName := string(swarmHash) + ".tar" + expectedContentDisposition := fmt.Sprintf("inline; filename=\"%s\"", expectedFileName) + if h := resp2.Header.Get("Content-Disposition"); h != expectedContentDisposition { + t.Fatalf("Content-Disposition header expected: %s, got: %s", expectedContentDisposition, h) + } + file, err := ioutil.TempFile("", "swarm-downloaded-tarball") if err != nil { t.Fatal(err) @@ -829,7 +815,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) { } func testBzzRootRedirect(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // create a manifest with some data at the root path @@ -884,7 +870,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { } func TestMethodsNotAllowed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() databytes := "bar" for _, c := range []struct { @@ -943,7 +929,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string } func TestGet(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for _, testCase := range []struct { @@ -1026,7 +1012,7 @@ func TestGet(t *testing.T) { } func TestModify(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() swarmClient := swarm.NewClient(srv.URL) @@ -1108,7 +1094,7 @@ func TestModify(t *testing.T) { res, body := httpDo(testCase.method, testCase.uri, reqBody, testCase.headers, testCase.verbose, t) if res.StatusCode != testCase.expectedStatusCode { - t.Fatalf("expected status code %d but got %d", testCase.expectedStatusCode, res.StatusCode) + t.Fatalf("expected status code %d but got %d, %s", testCase.expectedStatusCode, res.StatusCode, body) } if testCase.assertResponseBody != "" && !strings.Contains(body, testCase.assertResponseBody) { t.Log(body) @@ -1127,7 +1113,7 @@ func TestMultiPartUpload(t *testing.T) { // POST /bzz:/ Content-Type: multipart/form-data verbose := false // Setup Swarm - srv := testutil.NewTestSwarmServer(t, serverFunc, nil) + srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() url := fmt.Sprintf("%s/bzz:/", srv.URL) @@ -1158,7 +1144,7 @@ func TestMultiPartUpload(t *testing.T) { // TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver func TestBzzGetFileWithResolver(t *testing.T) { resolver := newTestResolveValidator("") - srv := testutil.NewTestSwarmServer(t, serverFunc, resolver) + srv := NewTestSwarmServer(t, serverFunc, resolver) defer srv.Close() fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} @@ -1219,19 +1205,25 @@ func TestBzzGetFileWithResolver(t *testing.T) { hash := common.HexToHash(string(swarmHash)) resolver.hash = &hash for _, v := range []struct { - addr string - path string - expectedStatusCode int + addr string + path string + expectedStatusCode int + expectedContentType string + expectedFileName string }{ { - addr: string(swarmHash), - path: fileNames[0], - expectedStatusCode: http.StatusOK, + addr: string(swarmHash), + path: fileNames[0], + expectedStatusCode: http.StatusOK, + expectedContentType: "text/plain", + expectedFileName: path.Base(fileNames[0]), }, { - addr: "somebogusensname", - path: fileNames[0], - expectedStatusCode: http.StatusOK, + addr: "somebogusensname", + path: fileNames[0], + expectedStatusCode: http.StatusOK, + expectedContentType: "text/plain", + expectedFileName: path.Base(fileNames[0]), }, } { req, err := http.NewRequest("GET", fmt.Sprintf(srv.URL+"/bzz:/%s/%s", v.addr, v.path), nil) @@ -1246,6 +1238,16 @@ func TestBzzGetFileWithResolver(t *testing.T) { if serverResponse.StatusCode != v.expectedStatusCode { t.Fatalf("expected %d, got %d", v.expectedStatusCode, serverResponse.StatusCode) } + + if h := serverResponse.Header.Get("Content-Type"); h != v.expectedContentType { + t.Fatalf("Content-Type header expected: %s, got %s", v.expectedContentType, h) + } + + expectedContentDisposition := fmt.Sprintf("inline; filename=\"%s\"", v.expectedFileName) + if h := serverResponse.Header.Get("Content-Disposition"); h != expectedContentDisposition { + t.Fatalf("Content-Disposition header expected: %s, got: %s", expectedContentDisposition, h) + } + } } diff --git a/swarm/testutil/http.go b/swarm/api/http/test_server.go similarity index 64% rename from swarm/testutil/http.go rename to swarm/api/http/test_server.go index 0748230329..9245c9c5b8 100644 --- a/swarm/testutil/http.go +++ b/swarm/api/http/test_server.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package testutil +package http import ( "io/ioutil" @@ -25,26 +25,13 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" ) type TestServer interface { ServeHTTP(http.ResponseWriter, *http.Request) } -// simulated timeProvider -type fakeTimeProvider struct { - currentTime uint64 -} - -func (f *fakeTimeProvider) Tick() { - f.currentTime++ -} - -func (f *fakeTimeProvider) Now() mru.Timestamp { - return mru.Timestamp{Time: f.currentTime} -} - func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, resolver api.Resolver) *TestSwarmServer { dir, err := ioutil.TempDir("", "swarm-storage-test") if err != nil { @@ -61,52 +48,50 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso } fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams()) - // mutable resources test setup - resourceDir, err := ioutil.TempDir("", "swarm-resource-test") + // Swarm feeds test setup + feedsDir, err := ioutil.TempDir("", "swarm-feeds-test") if err != nil { t.Fatal(err) } - fakeTimeProvider := &fakeTimeProvider{ - currentTime: 42, - } - mru.TimestampProvider = fakeTimeProvider - rhparams := &mru.HandlerParams{} - rh, err := mru.NewTestHandler(resourceDir, rhparams) + rhparams := &feed.HandlerParams{} + rh, err := feed.NewTestHandler(feedsDir, rhparams) if err != nil { t.Fatal(err) } a := api.NewAPI(fileStore, resolver, rh.Handler, nil) srv := httptest.NewServer(serverFunc(a)) - return &TestSwarmServer{ - Server: srv, - FileStore: fileStore, - dir: dir, - Hasher: storage.MakeHashFunc(storage.DefaultHash)(), - timestampProvider: fakeTimeProvider, + tss := &TestSwarmServer{ + Server: srv, + FileStore: fileStore, + dir: dir, + Hasher: storage.MakeHashFunc(storage.DefaultHash)(), cleanup: func() { srv.Close() rh.Close() os.RemoveAll(dir) - os.RemoveAll(resourceDir) + os.RemoveAll(feedsDir) }, + CurrentTime: 42, } + feed.TimestampProvider = tss + return tss } type TestSwarmServer struct { *httptest.Server - Hasher storage.SwarmHash - FileStore *storage.FileStore - dir string - cleanup func() - timestampProvider *fakeTimeProvider + Hasher storage.SwarmHash + FileStore *storage.FileStore + dir string + cleanup func() + CurrentTime uint64 } func (t *TestSwarmServer) Close() { t.cleanup() } -func (t *TestSwarmServer) GetCurrentTime() mru.Timestamp { - return t.timestampProvider.Now() +func (t *TestSwarmServer) Now() feed.Timestamp { + return feed.Timestamp{Time: t.CurrentTime} } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index d44ad2277c..890ed88bd4 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -27,14 +27,16 @@ import ( "strings" "time" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" ) const ( - ManifestType = "application/bzz-manifest+json" - ResourceContentType = "application/bzz-resource" + ManifestType = "application/bzz-manifest+json" + FeedContentType = "application/bzz-feed" manifestSizeLimit = 5 * 1024 * 1024 ) @@ -54,6 +56,7 @@ type ManifestEntry struct { ModTime time.Time `json:"mod_time,omitempty"` Status int `json:"status,omitempty"` Access *AccessEntry `json:"access,omitempty"` + Feed *feed.Feed `json:"feed,omitempty"` } // ManifestList represents the result of listing files in a manifest @@ -77,13 +80,13 @@ func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, return addr, err } -// Manifest hack for supporting Mutable Resource Updates from the bzz: scheme +// Manifest hack for supporting Swarm feeds from the bzz: scheme // see swarm/api/api.go:API.Get() for more information -func (a *API) NewResourceManifest(ctx context.Context, resourceAddr string) (storage.Address, error) { +func (a *API) NewFeedManifest(ctx context.Context, feed *feed.Feed) (storage.Address, error) { var manifest Manifest entry := ManifestEntry{ - Hash: resourceAddr, - ContentType: ResourceContentType, + Feed: feed, + ContentType: FeedContentType, } manifest.Entries = append(manifest.Entries, entry) data, err := json.Marshal(&manifest) @@ -554,7 +557,6 @@ func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manif if path != entry.Path { return nil, 0 } - pos = epl } } return nil, 0 diff --git a/swarm/api/testdata/test0/img/logo.png b/swarm/api/testdata/test0/img/logo.png index e0fb15ab33..9557f96053 100644 Binary files a/swarm/api/testdata/test0/img/logo.png and b/swarm/api/testdata/test0/img/logo.png differ diff --git a/swarm/api/uri.go b/swarm/api/uri.go index 808517088a..09cfa45020 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -86,7 +86,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-resource": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-feed": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -108,8 +108,8 @@ func Parse(rawuri string) (*URI, error) { } return uri, nil } -func (u *URI) Resource() bool { - return u.Scheme == "bzz-resource" +func (u *URI) Feed() bool { + return u.Scheme == "bzz-feed" } func (u *URI) Raw() bool { diff --git a/swarm/bmt/bmt_test.go b/swarm/bmt/bmt_test.go index 760aa11d8b..683ba4f5b2 100644 --- a/swarm/bmt/bmt_test.go +++ b/swarm/bmt/bmt_test.go @@ -18,10 +18,8 @@ package bmt import ( "bytes" - crand "crypto/rand" "encoding/binary" "fmt" - "io" "math/rand" "sync" "sync/atomic" @@ -29,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) // the actual data length generated (could be longer than max datalength of the BMT) @@ -116,14 +115,11 @@ func TestRefHasher(t *testing.T) { }) // run the tests - for _, x := range tests { + for i, x := range tests { for segmentCount := x.from; segmentCount <= x.to; segmentCount++ { for length := 1; length <= segmentCount*32; length++ { t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) { - data := make([]byte, length) - if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF { - t.Fatal(err) - } + data := testutil.RandomBytes(i, length) expected := x.expected(data) actual := NewRefHasher(sha3.NewKeccak256, segmentCount).Hash(data) if !bytes.Equal(actual, expected) { @@ -156,7 +152,7 @@ func TestHasherEmptyData(t *testing.T) { // tests sequential write with entire max size written in one go func TestSyncHasherCorrectness(t *testing.T) { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) hasher := sha3.NewKeccak256 size := hasher().Size() @@ -182,7 +178,7 @@ func TestSyncHasherCorrectness(t *testing.T) { // tests order-neutral concurrent writes with entire max size written in one go func TestAsyncCorrectness(t *testing.T) { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) hasher := sha3.NewKeccak256 size := hasher().Size() whs := []whenHash{first, last, random} @@ -236,7 +232,7 @@ func testHasherReuse(poolsize int, t *testing.T) { bmt := New(pool) for i := 0; i < 100; i++ { - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) n := rand.Intn(bmt.Size()) err := testHasherCorrectness(bmt, hasher, data, n, segmentCount) if err != nil { @@ -256,7 +252,7 @@ func TestBMTConcurrentUse(t *testing.T) { for i := 0; i < cycles; i++ { go func() { bmt := New(pool) - data := newData(BufferSize) + data := testutil.RandomBytes(1, BufferSize) n := rand.Intn(bmt.Size()) errc <- testHasherCorrectness(bmt, hasher, data, n, 128) }() @@ -290,7 +286,7 @@ func TestBMTWriterBuffers(t *testing.T) { defer pool.Drain(0) n := count * 32 bmt := New(pool) - data := newData(n) + data := testutil.RandomBytes(1, n) rbmt := NewRefHasher(hasher, count) refHash := rbmt.Hash(data) expHash := syncHash(bmt, nil, data) @@ -413,7 +409,7 @@ func BenchmarkPool(t *testing.B) { // benchmarks simple sha3 hash on chunks func benchmarkSHA3(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 h := hasher() @@ -432,7 +428,7 @@ func benchmarkSHA3(t *testing.B, n int) { func benchmarkBMTBaseline(t *testing.B, n int) { hasher := sha3.NewKeccak256 hashSize := hasher().Size() - data := newData(hashSize) + data := testutil.RandomBytes(1, hashSize) t.ReportAllocs() t.ResetTimer() @@ -456,7 +452,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) { // benchmarks BMT Hasher func benchmarkBMT(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool) @@ -470,12 +466,12 @@ func benchmarkBMT(t *testing.B, n int) { // benchmarks BMT hasher with asynchronous concurrent segment/section writes func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool).NewAsyncWriter(double) idxs, segments := splitAndShuffle(bmt.SectionSize(), data) - shuffle(len(idxs), func(i int, j int) { + rand.Shuffle(len(idxs), func(i int, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] }) @@ -488,7 +484,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { // benchmarks 100 concurrent bmt hashes with pool capacity func benchmarkPool(t *testing.B, poolsize, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 pool := NewTreePool(hasher, segmentCount, poolsize) cycles := 100 @@ -511,7 +507,7 @@ func benchmarkPool(t *testing.B, poolsize, n int) { // benchmarks the reference hasher func benchmarkRefHasher(t *testing.B, n int) { - data := newData(n) + data := testutil.RandomBytes(1, n) hasher := sha3.NewKeccak256 rbmt := NewRefHasher(hasher, 128) @@ -522,15 +518,6 @@ func benchmarkRefHasher(t *testing.B, n int) { } } -func newData(bufferSize int) []byte { - data := make([]byte, bufferSize) - _, err := io.ReadFull(crand.Reader, data) - if err != nil { - panic(err.Error()) - } - return data -} - // Hash hashes the data and the span using the bmt hasher func syncHash(h *Hasher, span, data []byte) []byte { h.ResetWithLength(span) @@ -553,7 +540,7 @@ func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) { section := data[i*secsize : end] segments = append(segments, section) } - shuffle(n, func(i int, j int) { + rand.Shuffle(n, func(i int, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] }) return idxs, segments @@ -594,29 +581,3 @@ func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, s } return <-c } - -// this is also in swarm/network_test.go -// shuffle pseudo-randomizes the order of elements. -// n is the number of elements. Shuffle panics if n < 0. -// swap swaps the elements with indexes i and j. -func shuffle(n int, swap func(i, j int)) { - if n < 0 { - panic("invalid argument to Shuffle") - } - - // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - // Shuffle really ought not be called with n that doesn't fit in 32 bits. - // Not only will it take a very long time, but with 2³¹! possible permutations, - // there's no way that any PRNG can have a big enough internal state to - // generate even a minuscule percentage of the possible permutations. - // Nevertheless, the right API signature accepts an int n, so handle it as best we can. - i := n - 1 - for ; i > 1<<31-1-1; i-- { - j := int(rand.Int63n(int64(i + 1))) - swap(i, j) - } - for ; i > 0; i-- { - j := int(rand.Int31n(int32(i + 1))) - swap(i, j) - } -} diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 87d9185505..460e31c4e9 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -20,20 +20,19 @@ package fuse import ( "bytes" - "crypto/rand" "flag" "fmt" "io" "io/ioutil" + "math/rand" "os" "path/filepath" "testing" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" - - "github.com/ethereum/go-ethereum/log" - + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -229,12 +228,6 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { } } -func getRandomBytes(size int) []byte { - contents := make([]byte, size) - rand.Read(contents) - return contents -} - func isDirEmpty(name string) bool { f, err := os.Open(name) if err != nil { @@ -328,22 +321,22 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "testMountDir") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)} - dat.files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)} - dat.files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)} - dat.files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)} - dat.files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)} - dat.files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)} - dat.files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)} - dat.files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)} - dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)} - dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)} - dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["2.txt"] = fileInfo{0711, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["3.txt"] = fileInfo{0622, 333, 444, testutil.RandomBytes(3, 100)} + dat.files["4.txt"] = fileInfo{0533, 333, 444, testutil.RandomBytes(4, 1024)} + dat.files["5.txt"] = fileInfo{0544, 333, 444, testutil.RandomBytes(5, 10)} + dat.files["6.txt"] = fileInfo{0555, 333, 444, testutil.RandomBytes(6, 10)} + dat.files["7.txt"] = fileInfo{0666, 333, 444, testutil.RandomBytes(7, 10)} + dat.files["8.txt"] = fileInfo{0777, 333, 333, testutil.RandomBytes(8, 10)} + dat.files["11.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(9, 10)} + dat.files["111.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(10, 10)} + dat.files["two/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(11, 10)} + dat.files["two/2/2.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(12, 10)} + dat.files["two/2./2.txt"] = fileInfo{0777, 444, 444, testutil.RandomBytes(13, 10)} + dat.files["twice/2.txt"] = fileInfo{0777, 444, 333, testutil.RandomBytes(14, 200)} + dat.files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, testutil.RandomBytes(15, 10240)} + dat.files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, testutil.RandomBytes(16, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -386,7 +379,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload1") dat.testMountDir = filepath.Join(dat.testDir, "max-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -396,7 +389,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload2") dat.testMountDir = filepath.Join(dat.testDir, "max-mount2") - dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -405,7 +398,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload3") dat.testMountDir = filepath.Join(dat.testDir, "max-mount3") - dat.files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -414,7 +407,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload4") dat.testMountDir = filepath.Join(dat.testDir, "max-mount4") - dat.files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["4.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -423,7 +416,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "max-upload5") dat.testMountDir = filepath.Join(dat.testDir, "max-mount5") - dat.files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -436,7 +429,7 @@ func (ta *testAPI) runMaxMounts(t *testing.T, toEncrypt bool) { if err != nil { t.Fatalf("Couldn't create upload dir 6: %v", err) } - dat.files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} testMountDir6 := filepath.Join(dat.testDir, "max-mount6") err = os.MkdirAll(testMountDir6, 0777) if err != nil { @@ -475,7 +468,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "remount-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -494,7 +487,7 @@ func (ta *testAPI) remount(t *testing.T, toEncrypt bool) { } // mount a different hash in already mounted point - dat.files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} testUploadDir2, err3 := addDir(dat.testDir, "remount-upload2") if err3 != nil { t.Fatalf("Error creating second upload dir: %v", err3) @@ -543,7 +536,7 @@ func (ta *testAPI) unmount(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -591,7 +584,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "ex-upload1") dat.testMountDir = filepath.Join(dat.testDir, "ex-mount1") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -609,7 +602,7 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T, toEncrypt bool) { //we need to manually close the file before mount for this test //but let's defer too in case of errors defer d.Close() - _, err = d.Write(getRandomBytes(10)) + _, err = d.Write(testutil.RandomBytes(1, 10)) if err != nil { t.Fatalf("Couldn't write to file: %v", err) } @@ -667,7 +660,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "seek-upload1") dat.testMountDir = filepath.Join(dat.testDir, "seek-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10240)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -733,9 +726,9 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "create-upload1") dat.testMountDir = filepath.Join(dat.testDir, "create-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -751,11 +744,7 @@ func (ta *testAPI) createNewFile(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("Opened file") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Could not rand read contents %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -815,7 +804,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "createinsidedir-upload") dat.testMountDir = filepath.Join(dat.testDir, "createinsidedir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -832,11 +821,7 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("File opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error filling random bytes into byte array %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("Content read") _, err = d.Write(contents) if err != nil { @@ -896,7 +881,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) dat.testUploadDir = filepath.Join(dat.testDir, "createinsidenewdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "createinsidenewdir-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -916,11 +901,7 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T, toEncrypt bool) } defer d.Close() log.Debug("File opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -976,9 +957,9 @@ func (ta *testAPI) removeExistingFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1036,9 +1017,9 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "remove-upload") dat.testMountDir = filepath.Join(dat.testDir, "remove-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["one/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["one/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1104,9 +1085,9 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "removenew-upload") dat.testMountDir = filepath.Join(dat.testDir, "removenew-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1127,11 +1108,7 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T, toEncrypt bool) { } defer d.Close() log.Debug("file opened") - contents := make([]byte, 11) - _, err = rand.Read(contents) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + contents := testutil.RandomBytes(1, 11) log.Debug("content read") _, err = d.Write(contents) if err != nil { @@ -1201,9 +1178,9 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "modifyfile-upload") dat.testMountDir = filepath.Join(dat.testDir, "modifyfile-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1357,9 +1334,9 @@ func (ta *testAPI) removeEmptyDir(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") dat.files = make(map[string]fileInfo) - dat.files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1406,9 +1383,9 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmdir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["two/five.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["two/six.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1480,12 +1457,12 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T, toEncrypt bool) { dat.testUploadDir = filepath.Join(dat.testDir, "rmsubdir-upload") dat.testMountDir = filepath.Join(dat.testDir, "rmsubdir-mount") dat.files = make(map[string]fileInfo) - dat.files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + dat.files["one/1.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(1, 10)} + dat.files["two/three/2.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(2, 10)} + dat.files["two/three/3.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(3, 10)} + dat.files["two/four/5.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(4, 10)} + dat.files["two/four/6.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(5, 10)} + dat.files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, testutil.RandomBytes(6, 10)} dat, err = ta.uploadAndMount(dat, t) if err != nil { @@ -1567,11 +1544,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { dat.testMountDir = filepath.Join(dat.testDir, "appendlargefile-mount") dat.files = make(map[string]fileInfo) - line1 := make([]byte, 10) - _, err = rand.Read(line1) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + line1 := testutil.RandomBytes(1, 10) dat.files["1.txt"] = fileInfo{0700, 333, 444, line1} @@ -1588,11 +1561,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T, toEncrypt bool) { } defer fd.Close() log.Debug("file opened") - line2 := make([]byte, 5) - _, err = rand.Read(line2) - if err != nil { - t.Fatalf("Error writing random bytes to byte array: %v", err) - } + line2 := testutil.RandomBytes(1, 5) log.Debug("line read") _, err = fd.Seek(int64(len(line1)), 0) if err != nil { diff --git a/swarm/network/README.md b/swarm/network/README.md index e9c9d30d0b..684ad0c8c0 100644 --- a/swarm/network/README.md +++ b/swarm/network/README.md @@ -37,7 +37,7 @@ Using the streamer logic, various stream types are easy to implement: * live session syncing * historical syncing * simple retrieve requests and deliveries -* mutable resource updates streams +* swarm feeds streams * receipting for finger pointing ## Syncing @@ -57,7 +57,7 @@ receipts for a deleted chunk easily to refute their challenge. - syncing should be resilient to cut connections, metadata should be persisted that keep track of syncing state across sessions, historical syncing state should survive restart - extra data structures to support syncing should be kept at minimum -- syncing is organized separately for chunk types (resource update v content chunk) +- syncing is not organized separately for chunk types (Swarm feed updates v regular content chunk) - various types of streams should have common logic abstracted Syncing is now entirely mediated by the localstore, ie., no processes or memory leaks due to network contention. diff --git a/swarm/network/fetcher.go b/swarm/network/fetcher.go index 5b4b61c7e2..6aed57e225 100644 --- a/swarm/network/fetcher.go +++ b/swarm/network/fetcher.go @@ -32,6 +32,8 @@ var searchTimeout = 1 * time.Second // Also used in stream delivery. var RequestTimeout = 10 * time.Second +var maxHopCount uint8 = 20 // maximum number of forwarded requests (hops), to make sure requests are not forwarded forever in peer loops + type RequestFunc func(context.Context, *Request) (*enode.ID, chan struct{}, error) // Fetcher is created when a chunk is not found locally. It starts a request handler loop once and @@ -44,7 +46,7 @@ type Fetcher struct { protoRequestFunc RequestFunc // request function fetcher calls to issue retrieve request for a chunk addr storage.Address // the address of the chunk to be fetched offerC chan *enode.ID // channel of sources (peer node id strings) - requestC chan struct{} + requestC chan uint8 // channel for incoming requests (with the hopCount value in it) skipCheck bool } @@ -53,6 +55,7 @@ type Request struct { Source *enode.ID // nodeID of peer to request from (can be nil) SkipCheck bool // whether to offer the chunk first or deliver directly peersToSkip *sync.Map // peers not to request chunk from (only makes sense if source is nil) + HopCount uint8 // number of forwarded requests (hops) } // NewRequest returns a new instance of Request based on chunk address skip check and @@ -113,7 +116,7 @@ func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher { addr: addr, protoRequestFunc: rf, offerC: make(chan *enode.ID), - requestC: make(chan struct{}), + requestC: make(chan uint8), skipCheck: skipCheck, } } @@ -136,7 +139,7 @@ func (f *Fetcher) Offer(ctx context.Context, source *enode.ID) { } // Request is called when an upstream peer request the chunk as part of `RetrieveRequestMsg`, or from a local request through FileStore, and the node does not have the chunk locally. -func (f *Fetcher) Request(ctx context.Context) { +func (f *Fetcher) Request(ctx context.Context, hopCount uint8) { // First we need to have this select to make sure that we return if context is done select { case <-ctx.Done(): @@ -144,10 +147,15 @@ func (f *Fetcher) Request(ctx context.Context) { default: } + if hopCount >= maxHopCount { + log.Debug("fetcher request hop count limit reached", "hops", hopCount) + return + } + // This select alone would not guarantee that we return of context is done, it could potentially // push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements) select { - case f.requestC <- struct{}{}: + case f.requestC <- hopCount + 1: case <-ctx.Done(): } } @@ -161,6 +169,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { waitC <-chan time.Time // timer channel sources []*enode.ID // known sources, ie. peers that offered the chunk requested bool // true if the chunk was actually requested + hopCount uint8 ) gone := make(chan *enode.ID) // channel to signal that a peer we requested from disconnected @@ -183,7 +192,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { doRequest = requested // incoming request - case <-f.requestC: + case hopCount = <-f.requestC: log.Trace("new request", "request addr", f.addr) // 2) chunk is requested, set requested flag // launch a request iff none been launched yet @@ -213,7 +222,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // need to issue a new request if doRequest { var err error - sources, err = f.doRequest(ctx, gone, peers, sources) + sources, err = f.doRequest(ctx, gone, peers, sources, hopCount) if err != nil { log.Info("unable to request", "request addr", f.addr, "err", err) } @@ -251,7 +260,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // * the peer's address is added to the set of peers to skip // * the peer's address is removed from prospective sources, and // * a go routine is started that reports on the gone channel if the peer is disconnected (or terminated their streamer) -func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSkip *sync.Map, sources []*enode.ID) ([]*enode.ID, error) { +func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSkip *sync.Map, sources []*enode.ID, hopCount uint8) ([]*enode.ID, error) { var i int var sourceID *enode.ID var quit chan struct{} @@ -260,6 +269,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki Addr: f.addr, SkipCheck: f.skipCheck, peersToSkip: peersToSkip, + HopCount: hopCount, } foundSource := false diff --git a/swarm/network/fetcher_test.go b/swarm/network/fetcher_test.go index b2316b0976..3a926f4758 100644 --- a/swarm/network/fetcher_test.go +++ b/swarm/network/fetcher_test.go @@ -33,7 +33,7 @@ type mockRequester struct { // requests []Request requestC chan *Request // when a request is coming it is pushed to requestC waitTimes []time.Duration // with waitTimes[i] you can define how much to wait on the ith request (optional) - ctr int //counts the number of requests + count int //counts the number of requests quitC chan struct{} } @@ -47,9 +47,9 @@ func newMockRequester(waitTimes ...time.Duration) *mockRequester { func (m *mockRequester) doRequest(ctx context.Context, request *Request) (*enode.ID, chan struct{}, error) { waitTime := time.Duration(0) - if m.ctr < len(m.waitTimes) { - waitTime = m.waitTimes[m.ctr] - m.ctr++ + if m.count < len(m.waitTimes) { + waitTime = m.waitTimes[m.count] + m.count++ } time.Sleep(waitTime) m.requestC <- request @@ -83,7 +83,7 @@ func TestFetcherSingleRequest(t *testing.T) { go fetcher.run(ctx, peersToSkip) rctx := context.Background() - fetcher.Request(rctx) + fetcher.Request(rctx, 0) select { case request := <-requester.requestC: @@ -100,6 +100,11 @@ func TestFetcherSingleRequest(t *testing.T) { t.Fatalf("request.peersToSkip does not contain peer returned by the request function") } + // hopCount in the forwarded request should be incremented + if request.HopCount != 1 { + t.Fatalf("Expected request.HopCount 1 got %v", request.HopCount) + } + // fetch should trigger a request, if it doesn't happen in time, test should fail case <-time.After(200 * time.Millisecond): t.Fatalf("fetch timeout") @@ -123,7 +128,7 @@ func TestFetcherCancelStopsFetcher(t *testing.T) { rctx, rcancel := context.WithTimeout(ctx, 100*time.Millisecond) defer rcancel() // we call Request with an active context - fetcher.Request(rctx) + fetcher.Request(rctx, 0) // fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening select { @@ -151,7 +156,7 @@ func TestFetcherCancelStopsRequest(t *testing.T) { rcancel() // we call Request with a cancelled context - fetcher.Request(rctx) + fetcher.Request(rctx, 0) // fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening select { @@ -162,7 +167,7 @@ func TestFetcherCancelStopsRequest(t *testing.T) { // if there is another Request with active context, there should be a request, because the fetcher itself is not cancelled rctx = context.Background() - fetcher.Request(rctx) + fetcher.Request(rctx, 0) select { case <-requester.requestC: @@ -200,7 +205,7 @@ func TestFetcherOfferUsesSource(t *testing.T) { // call Request after the Offer rctx = context.Background() - fetcher.Request(rctx) + fetcher.Request(rctx, 0) // there should be exactly 1 request coming from fetcher var request *Request @@ -241,7 +246,7 @@ func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) { // call Request first rctx := context.Background() - fetcher.Request(rctx) + fetcher.Request(rctx, 0) // there should be a request coming from fetcher var request *Request @@ -296,7 +301,7 @@ func TestFetcherRetryOnTimeout(t *testing.T) { // call the fetch function with an active context rctx := context.Background() - fetcher.Request(rctx) + fetcher.Request(rctx, 0) // after 100ms the first request should be initiated time.Sleep(100 * time.Millisecond) @@ -338,7 +343,7 @@ func TestFetcherFactory(t *testing.T) { fetcher := fetcherFactory.New(context.Background(), addr, peersToSkip) - fetcher.Request(context.Background()) + fetcher.Request(context.Background(), 0) // check if the created fetchFunction really starts a fetcher and initiates a request select { @@ -368,7 +373,7 @@ func TestFetcherRequestQuitRetriesRequest(t *testing.T) { go fetcher.run(ctx, peersToSkip) rctx := context.Background() - fetcher.Request(rctx) + fetcher.Request(rctx, 0) select { case <-requester.requestC: @@ -457,3 +462,26 @@ func TestRequestSkipPeerPermanent(t *testing.T) { t.Errorf("peer not skipped") } } + +func TestFetcherMaxHopCount(t *testing.T) { + requester := newMockRequester() + addr := make([]byte, 32) + fetcher := NewFetcher(addr, requester.doRequest, true) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + peersToSkip := &sync.Map{} + + go fetcher.run(ctx, peersToSkip) + + rctx := context.Background() + fetcher.Request(rctx, maxHopCount) + + // if hopCount is already at max no request should be initiated + select { + case <-requester.requestC: + t.Fatalf("cancelled fetcher initiated request") + case <-time.After(200 * time.Millisecond): + } +} diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 059c3dc96d..56adc5a8e9 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -70,6 +70,9 @@ func TestHiveStatePersistance(t *testing.T) { defer os.RemoveAll(dir) store, err := state.NewDBStore(dir) //start the hive with an empty dbstore + if err != nil { + t.Fatal(err) + } params := NewHiveParams() s, pp := newHiveTester(t, params, 5, store) @@ -90,6 +93,9 @@ func TestHiveStatePersistance(t *testing.T) { store.Close() persistedStore, err := state.NewDBStore(dir) //start the hive with an empty dbstore + if err != nil { + t.Fatal(err) + } s1, pp := newHiveTester(t, params, 1, persistedStore) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 55a0c6f13d..cd94741be3 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -261,7 +261,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // found among live peers, do nothing return v }) - if ins { + if ins && !p.BzzPeer.LightNode { a := newEntry(p.BzzAddr) a.conn = p // insert new online peer into addrs @@ -329,14 +329,18 @@ func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() var del bool - k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { - // v cannot be nil, must check otherwise we overwrite entry - if v == nil { - panic(fmt.Sprintf("connected peer not found %v", p)) - } + if !p.BzzPeer.LightNode { + k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { + // v cannot be nil, must check otherwise we overwrite entry + if v == nil { + panic(fmt.Sprintf("connected peer not found %v", p)) + } + del = true + return newEntry(p.BzzAddr) + }) + } else { del = true - return newEntry(p.BzzAddr) - }) + } if del { k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val { diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 903c8dbdac..d2e051f457 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -46,19 +46,19 @@ func newTestKademlia(b string) *Kademlia { return NewKademlia(base, params) } -func newTestKadPeer(k *Kademlia, s string) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k) +func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k) } func On(k *Kademlia, ons ...string) { for _, s := range ons { - k.On(newTestKadPeer(k, s)) + k.On(newTestKadPeer(k, s, false)) } } func Off(k *Kademlia, offs ...string) { for _, s := range offs { - k.Off(newTestKadPeer(k, s)) + k.Off(newTestKadPeer(k, s, false)) } } @@ -254,6 +254,56 @@ func TestSuggestPeerFindPeers(t *testing.T) { } +// a node should stay in the address book if it's removed from the kademlia +func TestOffEffectingAddressBookNormalNode(t *testing.T) { + k := newTestKademlia("00000000") + // peer added to kademlia + k.On(newTestKadPeer(k, "01000000", false)) + // peer should be in the address book + if k.addrs.Size() != 1 { + t.Fatal("known peer addresses should contain 1 entry") + } + // peer should be among live connections + if k.conns.Size() != 1 { + t.Fatal("live peers should contain 1 entry") + } + // remove peer from kademlia + k.Off(newTestKadPeer(k, "01000000", false)) + // peer should be in the address book + if k.addrs.Size() != 1 { + t.Fatal("known peer addresses should contain 1 entry") + } + // peer should not be among live connections + if k.conns.Size() != 0 { + t.Fatal("live peers should contain 0 entry") + } +} + +// a light node should not be in the address book +func TestOffEffectingAddressBookLightNode(t *testing.T) { + k := newTestKademlia("00000000") + // light node peer added to kademlia + k.On(newTestKadPeer(k, "01000000", true)) + // peer should not be in the address book + if k.addrs.Size() != 0 { + t.Fatal("known peer addresses should contain 0 entry") + } + // peer should be among live connections + if k.conns.Size() != 1 { + t.Fatal("live peers should contain 1 entry") + } + // remove peer from kademlia + k.Off(newTestKadPeer(k, "01000000", true)) + // peer should not be in the address book + if k.addrs.Size() != 0 { + t.Fatal("known peer addresses should contain 0 entry") + } + // peer should not be among live connections + if k.conns.Size() != 0 { + t.Fatal("live peers should contain 0 entry") + } +} + func TestSuggestPeerRetries(t *testing.T) { k := newTestKademlia("00000000") k.RetryInterval = int64(300 * time.Millisecond) // cycle diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 4b83c7a278..f0d2666288 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -50,10 +50,6 @@ type testStore struct { values map[string][]byte } -func newTestStore() *testStore { - return &testStore{values: make(map[string][]byte)} -} - func (t *testStore) Load(key string) ([]byte, error) { t.Lock() defer t.Unlock() @@ -157,17 +153,7 @@ func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) * // should test handshakes in one exchange? parallelisation func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptest.Disconnect) error { - var peers []enode.ID - id := rhs.Addr.ID() - if len(disconnects) > 0 { - for _, d := range disconnects { - peers = append(peers, d.Peer) - } - } else { - peers = []enode.ID{id} - } - - if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, id)...); err != nil { + if err := s.TestExchanges(HandshakeMsgExchange(lhs, rhs, rhs.Addr.ID())...); err != nil { return err } diff --git a/swarm/network/simulation/bucket_test.go b/swarm/network/simulation/bucket_test.go index 461d99825c..69df19bfe4 100644 --- a/swarm/network/simulation/bucket_test.go +++ b/swarm/network/simulation/bucket_test.go @@ -94,7 +94,7 @@ func TestServiceBucket(t *testing.T) { t.Fatalf("expected %q, got %q", customValue, s) } - v, ok = sim.NodeItem(id2, customKey) + _, ok = sim.NodeItem(id2, customKey) if ok { t.Fatal("bucket item should not be found") } @@ -119,7 +119,7 @@ func TestServiceBucket(t *testing.T) { t.Fatalf("expected %q, got %q", testValue+id1.String(), s) } - v, ok = items[id2] + _, ok = items[id2] if ok { t.Errorf("node 2 item should not be found") } diff --git a/swarm/network/simulation/events.go b/swarm/network/simulation/events.go index 594d36225c..d73c3af4ee 100644 --- a/swarm/network/simulation/events.go +++ b/swarm/network/simulation/events.go @@ -20,16 +20,18 @@ import ( "context" "sync" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" ) // PeerEvent is the type of the channel returned by Simulation.PeerEvents. type PeerEvent struct { // NodeID is the ID of node that the event is caught on. NodeID enode.ID + // PeerID is the ID of the peer node that the event is caught on. + PeerID enode.ID // Event is the event that is caught. - Event *p2p.PeerEvent + Event *simulations.Event // Error is the error that may have happened during event watching. Error error } @@ -37,9 +39,13 @@ type PeerEvent struct { // PeerEventsFilter defines a filter on PeerEvents to exclude messages with // defined properties. Use PeerEventsFilter methods to set required options. type PeerEventsFilter struct { - t *p2p.PeerEventType - protocol *string - msgCode *uint64 + eventType simulations.EventType + + connUp *bool + + msgReceive *bool + protocol *string + msgCode *uint64 } // NewPeerEventsFilter returns a new PeerEventsFilter instance. @@ -47,20 +53,48 @@ func NewPeerEventsFilter() *PeerEventsFilter { return &PeerEventsFilter{} } -// Type sets the filter to only one peer event type. -func (f *PeerEventsFilter) Type(t p2p.PeerEventType) *PeerEventsFilter { - f.t = &t +// Connect sets the filter to events when two nodes connect. +func (f *PeerEventsFilter) Connect() *PeerEventsFilter { + f.eventType = simulations.EventTypeConn + b := true + f.connUp = &b + return f +} + +// Drop sets the filter to events when two nodes disconnect. +func (f *PeerEventsFilter) Drop() *PeerEventsFilter { + f.eventType = simulations.EventTypeConn + b := false + f.connUp = &b + return f +} + +// ReceivedMessages sets the filter to only messages that are received. +func (f *PeerEventsFilter) ReceivedMessages() *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg + b := true + f.msgReceive = &b + return f +} + +// SentMessages sets the filter to only messages that are sent. +func (f *PeerEventsFilter) SentMessages() *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg + b := false + f.msgReceive = &b return f } // Protocol sets the filter to only one message protocol. func (f *PeerEventsFilter) Protocol(p string) *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg f.protocol = &p return f } // MsgCode sets the filter to only one msg code. func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter { + f.eventType = simulations.EventTypeMsg f.msgCode = &c return f } @@ -80,19 +114,8 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ... go func(id enode.ID) { defer s.shutdownWG.Done() - client, err := s.Net.GetNode(id).Client() - if err != nil { - subsWG.Done() - eventC <- PeerEvent{NodeID: id, Error: err} - return - } - events := make(chan *p2p.PeerEvent) - sub, err := client.Subscribe(ctx, "admin", events, "peerEvents") - if err != nil { - subsWG.Done() - eventC <- PeerEvent{NodeID: id, Error: err} - return - } + events := make(chan *simulations.Event) + sub := s.Net.Events().Subscribe(events) defer sub.Unsubscribe() subsWG.Done() @@ -110,28 +133,55 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []enode.ID, filters ... case <-s.Done(): return case e := <-events: + // ignore control events + if e.Control { + continue + } match := len(filters) == 0 // if there are no filters match all events for _, f := range filters { - if f.t != nil && *f.t != e.Type { - continue + if f.eventType == simulations.EventTypeConn && e.Conn != nil { + if *f.connUp != e.Conn.Up { + continue + } + // all connection filter parameters matched, break the loop + match = true + break } - if f.protocol != nil && *f.protocol != e.Protocol { - continue + if f.eventType == simulations.EventTypeMsg && e.Msg != nil { + if f.msgReceive != nil && *f.msgReceive != e.Msg.Received { + continue + } + if f.protocol != nil && *f.protocol != e.Msg.Protocol { + continue + } + if f.msgCode != nil && *f.msgCode != e.Msg.Code { + continue + } + // all message filter parameters matched, break the loop + match = true + break } - if f.msgCode != nil && e.MsgCode != nil && *f.msgCode != *e.MsgCode { - continue + } + var peerID enode.ID + switch e.Type { + case simulations.EventTypeConn: + peerID = e.Conn.One + if peerID == id { + peerID = e.Conn.Other + } + case simulations.EventTypeMsg: + peerID = e.Msg.One + if peerID == id { + peerID = e.Msg.Other } - // all filter parameters matched, break the loop - match = true - break } if match { select { - case eventC <- PeerEvent{NodeID: id, Event: e}: + case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Event: e}: case <-ctx.Done(): if err := ctx.Err(); err != nil { select { - case eventC <- PeerEvent{NodeID: id, Error: err}: + case eventC <- PeerEvent{NodeID: id, PeerID: peerID, Error: err}: case <-s.Done(): } } diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index 84b0634b45..bacc64d53a 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" @@ -87,7 +86,7 @@ func ExampleSimulation_PeerEvents() { log.Error("peer event", "err", e.Error) continue } - log.Info("peer event", "node", e.NodeID, "peer", e.Event.Peer, "msgcode", e.Event.MsgCode) + log.Info("peer event", "node", e.NodeID, "peer", e.PeerID, "type", e.Event.Type) } }() } @@ -100,7 +99,7 @@ func ExampleSimulation_PeerEvents_disconnections() { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { @@ -109,7 +108,7 @@ func ExampleSimulation_PeerEvents_disconnections() { log.Error("peer drop", "err", d.Error) continue } - log.Warn("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Warn("peer drop", "node", d.NodeID, "peer", d.PeerID) } }() } @@ -124,8 +123,8 @@ func ExampleSimulation_PeerEvents_multipleFilters() { context.Background(), sim.NodeIDs(), // Watch when bzz messages 1 and 4 are received. - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(1), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("bzz").MsgCode(4), + simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(1), + simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("bzz").MsgCode(4), ) go func() { @@ -134,7 +133,7 @@ func ExampleSimulation_PeerEvents_multipleFilters() { log.Error("bzz message", "err", m.Error) continue } - log.Info("bzz message", "node", m.NodeID, "peer", m.Event.Peer) + log.Info("bzz message", "node", m.NodeID, "peer", m.PeerID) } }() } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index d11eabf95f..cd5456b73e 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -85,11 +85,12 @@ func getDbStore(nodeID string) (*state.DBStore, error) { } var ( - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") - initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") - snapshotFile = flag.String("snapshot", "", "create snapshot") - loglevel = flag.Int("loglevel", 3, "verbosity of logs") - rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") + nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") + snapshotFile = flag.String("snapshot", "", "path to create snapshot file in") + loglevel = flag.Int("loglevel", 3, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") + serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery") ) func init() { @@ -125,22 +126,6 @@ func BenchmarkDiscovery_64_4(b *testing.B) { benchmarkDiscovery(b, 64, 4) } func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) } func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) } -func TestDiscoverySimulationDockerAdapter(t *testing.T) { - testDiscoverySimulationDockerAdapter(t, *nodeCount, *initCount) -} - -func testDiscoverySimulationDockerAdapter(t *testing.T, nodes, conns int) { - adapter, err := adapters.NewDockerAdapter() - if err != nil { - if err == adapters.ErrLinuxOnly { - t.Skip(err) - } else { - t.Fatal(err) - } - } - testDiscoverySimulation(t, nodes, conns, adapter) -} - func TestDiscoverySimulationExecAdapter(t *testing.T) { testDiscoverySimulationExecAdapter(t, *nodeCount, *initCount) } @@ -322,7 +307,25 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul } if *snapshotFile != "" { - snap, err := net.Snapshot() + var err error + var snap *simulations.Snapshot + if len(*serviceOverride) > 0 { + var addServices []string + var removeServices []string + for _, osvc := range strings.Split(*serviceOverride, ",") { + if strings.Index(osvc, "+") == 0 { + addServices = append(addServices, osvc[1:]) + } else if strings.Index(osvc, "-") == 0 { + removeServices = append(removeServices, osvc[1:]) + } else { + panic("stick to the rules, you know what they are") + } + } + snap, err = net.SnapshotWithServices(addServices, removeServices) + } else { + snap, err = net.Snapshot() + } + if err != nil { return nil, errors.New("no shapshot dude") } @@ -545,8 +548,7 @@ func triggerChecks(trigger chan enode.ID, net *simulations.Network, id enode.ID) } func newService(ctx *adapters.ServiceContext) (node.Service, error) { - node := enode.NewV4(&ctx.Config.PrivateKey.PublicKey, adapters.ExternalIP(), int(ctx.Config.Port), int(ctx.Config.Port)) - addr := network.NewAddr(node) + addr := network.NewAddr(ctx.Config.Node()) kp := network.NewKadParams() kp.MinProxBinSize = testMinProxBinSize diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index deb7e98155..c5f1fa176a 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -18,7 +18,6 @@ package stream import ( "context" - crand "crypto/rand" "errors" "flag" "fmt" @@ -40,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" ) @@ -84,7 +84,7 @@ func createGlobalStore() (string, *mockdb.GlobalStore, error) { return globalStoreDir, globalStore, nil } -func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { +func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { // setup addr := network.RandomAddr() // tested peers peer address to := network.NewKademlia(addr.OAddr, network.NewKadParams()) @@ -114,7 +114,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora delivery := NewDelivery(to, netStore) netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), nil) + streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil) teardown := func() { streamer.Close() removeDataDir() @@ -230,12 +230,7 @@ func generateRandomFile() (string, error) { //generate a random file size between minFileSize and maxFileSize fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize)) - b := make([]byte, fileSize*1024) - _, err := crand.Read(b) - if err != nil { - log.Error("Error generating random file.", "err", err) - return "", err - } + b := testutil.RandomBytes(1, fileSize*1024) return string(b), nil } diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 431136ab1d..0109fbdef2 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -96,6 +96,11 @@ func (s *SwarmChunkServer) processDeliveries() { } } +// SessionIndex returns zero in all cases for SwarmChunkServer. +func (s *SwarmChunkServer) SessionIndex() (uint64, error) { + return 0, nil +} + // SetNextBatch func (s *SwarmChunkServer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { select { @@ -128,6 +133,7 @@ func (s *SwarmChunkServer) GetData(ctx context.Context, key []byte) ([]byte, err type RetrieveRequestMsg struct { Addr storage.Address SkipCheck bool + HopCount uint8 } func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req *RetrieveRequestMsg) error { @@ -140,7 +146,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * "retrieve.request") defer osp.Finish() - s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", false)) + s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", true)) if err != nil { return err } @@ -148,7 +154,9 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * var cancel func() // TODO: do something with this hardcoded timeout, maybe use TTL in the future - ctx, cancel = context.WithTimeout(context.WithValue(ctx, "peer", sp.ID().String()), network.RequestTimeout) + ctx = context.WithValue(ctx, "peer", sp.ID().String()) + ctx = context.WithValue(ctx, "hopcount", req.HopCount) + ctx, cancel = context.WithTimeout(ctx, network.RequestTimeout) go func() { select { @@ -165,7 +173,8 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return } if req.SkipCheck { - err = sp.Deliver(ctx, chunk, s.priority) + syncing := false + err = sp.Deliver(ctx, chunk, s.priority, syncing) if err != nil { log.Warn("ERROR in handleRetrieveRequestMsg", "err", err) } @@ -181,12 +190,22 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * return nil } +//Chunk delivery always uses the same message type.... type ChunkDeliveryMsg struct { Addr storage.Address SData []byte // the stored chunk Data (incl size) peer *Peer // set in handleChunkDeliveryMsg } +//...but swap accounting needs to disambiguate if it is a delivery for syncing or for retrieval +//as it decides based on message type if it needs to account for this message or not + +//defines a chunk delivery for retrieval (with accounting) +type ChunkDeliveryMsgRetrieval ChunkDeliveryMsg + +//defines a chunk delivery for syncing (without accounting) +type ChunkDeliveryMsgSyncing ChunkDeliveryMsg + // TODO: Fix context SNAFU func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *ChunkDeliveryMsg) error { var osp opentracing.Span @@ -226,7 +245,10 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( } else { d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int, nn bool) bool { id := p.ID() - // TODO: skip light nodes that do not accept retrieve requests + if p.LightNode { + // skip light nodes + return true + } if req.SkipPeer(id.String()) { log.Trace("Delivery.RequestFromPeers: skip peer", "peer id", id) return true @@ -247,6 +269,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( err := sp.SendPriority(ctx, &RetrieveRequestMsg{ Addr: req.Addr, SkipCheck: req.SkipCheck, + HopCount: req.HopCount, }, Top) if err != nil { return nil, nil, err diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 9fb90eebaf..a6173a3892 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -19,9 +19,7 @@ package stream import ( "bytes" "context" - crand "crypto/rand" "fmt" - "io" "os" "sync" "testing" @@ -29,17 +27,26 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/network" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) +//Tests initializing a retrieve request func TestStreamerRetrieveRequest(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + regOpts := &RegistryOptions{ + Retrieval: RetrievalClientOnly, + Syncing: SyncingDisabled, + } + tester, streamer, _, teardown, err := newStreamerTester(t, regOpts) defer teardown() if err != nil { t.Fatal(err) @@ -55,10 +62,21 @@ func TestStreamerRetrieveRequest(t *testing.T) { ) streamer.delivery.RequestFromPeers(ctx, req) + stream := NewStream(swarmChunkServerStreamName, "", true) + err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", Expects: []p2ptest.Expect{ - { + { //start expecting a subscription for RETRIEVE_REQUEST due to `RetrievalClientOnly` + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + { //expect a retrieve request message for the given hash Code: 5, Msg: &RetrieveRequestMsg{ Addr: hash0[:], @@ -74,8 +92,13 @@ func TestStreamerRetrieveRequest(t *testing.T) { } } +//Test requesting a chunk from a peer then issuing a "empty" OfferedHashesMsg (no hashes available yet) +//Should time out as the peer does not have the chunk (no syncing happened previously) func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalEnabled, + Syncing: SyncingDisabled, //do no syncing + }) defer teardown() if err != nil { t.Fatal(err) @@ -87,16 +110,31 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { peer := streamer.getPeer(node.ID()) + stream := NewStream(swarmChunkServerStreamName, "", true) + //simulate pre-subscription to RETRIEVE_REQUEST stream on peer peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{ - Stream: NewStream(swarmChunkServerStreamName, "", false), + Stream: stream, History: nil, Priority: Top, }) + //test the exchange err = tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { //first expect a subscription to the RETRIEVE_REQUEST stream + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ - { + { //then the actual RETRIEVE_REQUEST.... Code: 5, Msg: &RetrieveRequestMsg{ Addr: chunk.Address()[:], @@ -105,7 +143,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }, Expects: []p2ptest.Expect{ - { + { //to which the peer responds with offered hashes Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: nil, @@ -118,7 +156,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { }, }) - expectedError := `exchange #0 "RetrieveRequestMsg": timed out` + //should fail with a timeout as the peer we are requesting + //the chunk from does not have the chunk + expectedError := `exchange #1 "RetrieveRequestMsg": timed out` if err == nil || err.Error() != expectedError { t.Fatalf("Expected error %v, got %v", expectedError, err) } @@ -127,16 +167,20 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // upstream request server receives a retrieve Request and responds with // offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { - tester, streamer, localStore, teardown, err := newStreamerTester(t) + tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalEnabled, + Syncing: SyncingDisabled, + }) defer teardown() if err != nil { t.Fatal(err) } node := tester.Nodes[0] + peer := streamer.getPeer(node.ID()) - stream := NewStream(swarmChunkServerStreamName, "", false) + stream := NewStream(swarmChunkServerStreamName, "", true) peer.handleSubscribeMsg(context.TODO(), &SubscribeMsg{ Stream: stream, @@ -152,6 +196,18 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } err = tester.TestExchanges(p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: nil, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ Label: "RetrieveRequestMsg", Triggers: []p2ptest.Trigger{ { @@ -220,8 +276,91 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } } +// if there is one peer in the Kademlia, RequestFromPeers should return it +func TestRequestFromPeers(t *testing.T) { + dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") + + addr := network.RandomAddr() + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + delivery := NewDelivery(to, nil) + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + peer := network.NewPeer(&network.BzzPeer{ + BzzAddr: network.RandomAddr(), + LightNode: false, + Peer: protocolsPeer, + }, to) + to.On(peer) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil) + + // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished + sp := &Peer{ + Peer: protocolsPeer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: r, + } + r.setPeer(sp) + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + ctx := context.Background() + id, _, err := delivery.RequestFromPeers(ctx, req) + + if err != nil { + t.Fatal(err) + } + if *id != dummyPeerID { + t.Fatalf("Expected an id, got %v", id) + } +} + +// RequestFromPeers should not return light nodes +func TestRequestFromPeersWithLightNode(t *testing.T) { + dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") + + addr := network.RandomAddr() + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + delivery := NewDelivery(to, nil) + + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + // setting up a lightnode + peer := network.NewPeer(&network.BzzPeer{ + BzzAddr: network.RandomAddr(), + LightNode: true, + Peer: protocolsPeer, + }, to) + to.On(peer) + r := NewRegistry(addr.ID(), delivery, nil, nil, nil, nil) + // an empty priorityQueue has to be created to prevent a goroutine being called after the test has finished + sp := &Peer{ + Peer: protocolsPeer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: r, + } + r.setPeer(sp) + + req := network.NewRequest( + storage.Address(hash0[:]), + true, + &sync.Map{}, + ) + + ctx := context.Background() + // making a request which should return with "no peer found" + _, _, err := delivery.RequestFromPeers(ctx, req) + + expectedError := "no peer found" + if err.Error() != expectedError { + t.Fatalf("expected '%v', got %v", expectedError, err) + } +} + func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { - tester, streamer, localStore, teardown, err := newStreamerTester(t) + tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, + }) defer teardown() if err != nil { t.Fatal(err) @@ -235,6 +374,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { node := tester.Nodes[0] + //subscribe to custom stream stream := NewStream("foo", "", true) err = streamer.Subscribe(node.ID(), stream, NewRange(5, 8), Top) if err != nil { @@ -247,7 +387,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", Expects: []p2ptest.Expect{ - { + { //first expect subscription to the custom stream... Code: 4, Msg: &SubscribeMsg{ Stream: stream, @@ -261,7 +401,8 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { p2ptest.Exchange{ Label: "ChunkDelivery message", Triggers: []p2ptest.Trigger{ - { + { //...then trigger a chunk delivery for the given chunk from peer in order for + //local node to get the chunk delivered Code: 6, Msg: &ChunkDeliveryMsg{ Addr: chunkKey, @@ -337,7 +478,9 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, - }) + Syncing: SyncingDisabled, + Retrieval: RetrievalEnabled, + }, nil) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) @@ -386,7 +529,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck //now we can actually upload a (random) file to the round-robin store size := chunkCount * chunkSize log.Debug("Storing data to file store") - fileHash, wait, err := roundRobinFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) // wait until all chunks stored if err != nil { return err @@ -401,20 +544,6 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck return err } - //each of the nodes (except pivot node) subscribes to the stream of the next node - for j, node := range nodeIDs[0 : nodes-1] { - sid := nodeIDs[j+1] - item, ok := sim.NodeItem(node, bucketKeyRegistry) - if !ok { - return fmt.Errorf("No registry") - } - registry := item.(*Registry) - err = registry.Subscribe(sid, NewStream(swarmChunkServerStreamName, "", false), NewRange(0, 0), Top) - if err != nil { - return err - } - } - //get the pivot node's filestore item, ok := sim.NodeItem(*sim.PivotNodeID(), bucketKeyFileStore) if !ok { @@ -436,13 +565,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } @@ -523,9 +652,10 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ SkipCheck: skipCheck, - DoSync: true, + Syncing: SyncingDisabled, + Retrieval: RetrievalDisabled, SyncUpdateDelay: 0, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) @@ -567,13 +697,13 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) b.Fatal(d.Error) } } @@ -588,7 +718,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip for i := 0; i < chunkCount; i++ { // create actual size real chunks ctx := context.TODO() - hash, wait, err := remoteFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize), false) + hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false) if err != nil { b.Fatalf("expected no error. got %v", err) } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 2692594238..defb6df50a 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -18,10 +18,8 @@ package stream import ( "context" - crand "crypto/rand" "encoding/binary" "fmt" - "io" "os" "sync" "testing" @@ -29,13 +27,13 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestIntervalsLive(t *testing.T) { @@ -83,8 +81,10 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, SkipCheck: skipCheck, - }) + }, nil) bucket.Store(bucketKeyRegistry, r) r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) { @@ -128,7 +128,8 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { fileStore := item.(*storage.FileStore) size := chunkCount * chunkSize - _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + + _, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) if err != nil { log.Error("Store error: %v", "err", err) t.Fatal(err) @@ -152,7 +153,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) @@ -163,7 +164,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } @@ -345,8 +346,6 @@ func (c *testExternalClient) BatchDone(Stream, uint64, []byte, []byte) func() (* func (c *testExternalClient) Close() {} -const testExternalServerBatchSize = 10 - type testExternalServer struct { t string keyFunc func(key []byte, index uint64) @@ -366,17 +365,11 @@ func newTestExternalServer(t string, sessionAt, maxKeys uint64, keyFunc func(key } } +func (s *testExternalServer) SessionIndex() (uint64, error) { + return s.sessionAt, nil +} + func (s *testExternalServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { - if from == 0 && to == 0 { - from = s.sessionAt - to = s.sessionAt + testExternalServerBatchSize - } - if to-from > testExternalServerBatchSize { - to = from + testExternalServerBatchSize - 1 - } - if from >= s.maxKeys && to > s.maxKeys { - return nil, 0, 0, nil, io.EOF - } if to > s.maxKeys { to = s.maxKeys } diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go new file mode 100644 index 0000000000..65cde2411c --- /dev/null +++ b/swarm/network/stream/lightnode_test.go @@ -0,0 +1,214 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package stream + +import ( + "testing" + + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" +) + +// This test checks the default behavior of the server, that is +// when it is serving Retrieve requests. +func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { + registryOptions := &RegistryOptions{ + Retrieval: RetrievalClientOnly, + Syncing: SyncingDisabled, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + stream := NewStream(swarmChunkServerStreamName, "", false) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "SubscribeMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + }, + Peer: node.ID(), + }, + }, + }) + if err != nil { + t.Fatalf("Got %v", err) + } + + err = tester.TestDisconnected(&p2ptest.Disconnect{Peer: node.ID()}) + if err == nil || err.Error() != "timed out waiting for peers to disconnect" { + t.Fatalf("Expected no disconnect, got %v", err) + } +} + +// This test checks the Lightnode behavior of server, when serving Retrieve +// requests are disabled +func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { + registryOptions := &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + stream := NewStream(swarmChunkServerStreamName, "", false) + + err = tester.TestExchanges( + p2ptest.Exchange{ + Label: "SubscribeMsg", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream RETRIEVE_REQUEST not registered", + }, + Peer: node.ID(), + }, + }, + }) + if err != nil { + t.Fatalf("Got %v", err) + } +} + +// This test checks the default behavior of the server, that is +// when syncing is enabled. +func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { + registryOptions := &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + syncStream := NewStream("SYNC", FormatSyncBinKey(1), false) + + err = tester.TestExchanges( + p2ptest.Exchange{ + Label: "RequestSubscription", + Triggers: []p2ptest.Trigger{ + { + Code: 8, + Msg: &RequestSubscriptionMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatalf("Got %v", err) + } +} + +// This test checks the Lightnode behavior of the server, that is +// when syncing is disabled. +func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) { + registryOptions := &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, + } + tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + defer teardown() + if err != nil { + t.Fatal(err) + } + + node := tester.Nodes[0] + + syncStream := NewStream("SYNC", FormatSyncBinKey(1), false) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RequestSubscription", + Triggers: []p2ptest.Trigger{ + { + Code: 8, + Msg: &RequestSubscriptionMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream SYNC not registered", + }, + Peer: node.ID(), + }, + }, + }, p2ptest.Exchange{ + Label: "RequestSubscription", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: syncStream, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: "stream SYNC not registered", + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatalf("Got %v", err) + } +} diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 0332322f49..eb1b2983e1 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -26,7 +26,7 @@ import ( bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" ) var syncBatchTimeout = 30 * time.Second @@ -76,7 +76,16 @@ type RequestSubscriptionMsg struct { func (p *Peer) handleRequestSubscription(ctx context.Context, req *RequestSubscriptionMsg) (err error) { log.Debug(fmt.Sprintf("handleRequestSubscription: streamer %s to subscribe to %s with stream %s", p.streamer.addr, p.ID(), req.Stream)) - return p.streamer.Subscribe(p.ID(), req.Stream, req.History, req.Priority) + if err = p.streamer.Subscribe(p.ID(), req.Stream, req.History, req.Priority); err != nil { + // The error will be sent as a subscribe error message + // and will not be returned as it will prevent any new message + // exchange between peers over p2p. Instead, error will be returned + // only if there is one from sending subscribe error message. + err = p.Send(ctx, SubscribeErrorMsg{ + Error: err.Error(), + }) + } + return err } func (p *Peer) handleSubscribeMsg(ctx context.Context, req *SubscribeMsg) (err error) { @@ -84,11 +93,13 @@ func (p *Peer) handleSubscribeMsg(ctx context.Context, req *SubscribeMsg) (err e defer func() { if err != nil { - if e := p.Send(context.TODO(), SubscribeErrorMsg{ + // The error will be sent as a subscribe error message + // and will not be returned as it will prevent any new message + // exchange between peers over p2p. Instead, error will be returned + // only if there is one from sending subscribe error message. + err = p.Send(context.TODO(), SubscribeErrorMsg{ Error: err.Error(), - }); e != nil { - log.Error("send stream subscribe error message", "err", err) - } + }) } }() @@ -147,6 +158,7 @@ type SubscribeErrorMsg struct { } func (p *Peer) handleSubscribeErrorMsg(req *SubscribeErrorMsg) (err error) { + //TODO the error should be channeled to whoever calls the subscribe return fmt.Errorf("subscribe to peer %s: %v", p.ID(), req.Error) } @@ -195,10 +207,16 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg if err != nil { return err } + hashes := req.Hashes - want, err := bv.New(len(hashes) / HashSize) + lenHashes := len(hashes) + if lenHashes%HashSize != 0 { + return fmt.Errorf("error invalid hashes length (len: %v)", lenHashes) + } + + want, err := bv.New(lenHashes / HashSize) if err != nil { - return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err) + return fmt.Errorf("error initiaising bitvector of length %v: %v", lenHashes/HashSize, err) } ctr := 0 @@ -206,7 +224,7 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg ctx, cancel := context.WithTimeout(ctx, syncBatchTimeout) ctx = context.WithValue(ctx, "source", p.ID().String()) - for i := 0; i < len(hashes); i += HashSize { + for i := 0; i < lenHashes; i += HashSize { hash := hashes[i : i+HashSize] if wait := c.NeedData(ctx, hash); wait != nil { @@ -339,7 +357,8 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err) } chunk := storage.NewChunk(hash, data) - if err := p.Deliver(ctx, chunk, s.priority); err != nil { + syncing := true + if err := p.Deliver(ctx, chunk, s.priority, syncing); err != nil { return err } } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 5fdaa7b878..4bccf56f5d 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -18,6 +18,7 @@ package stream import ( "context" + "errors" "fmt" "sync" "time" @@ -46,6 +47,10 @@ func (e *notFoundError) Error() string { return fmt.Sprintf("%s not found for stream %q", e.t, e.s) } +// ErrMaxPeerServers will be returned if peer server limit is reached. +// It will be sent in the SubscribeErrorMsg. +var ErrMaxPeerServers = errors.New("max peer servers") + // Peer is the Peer extension for the streaming protocol type Peer struct { *protocols.Peer @@ -123,17 +128,34 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { } // Deliver sends a storeRequestMsg protocol message to the peer -func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8) error { +// Depending on the `syncing` parameter we send different message types +func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8, syncing bool) error { var sp opentracing.Span + var msg interface{} + + spanName := "send.chunk.delivery" + + //we send different types of messages if delivery is for syncing or retrievals, + //even if handling and content of the message are the same, + //because swap accounting decides which messages need accounting based on the message type + if syncing { + msg = &ChunkDeliveryMsgSyncing{ + Addr: chunk.Address(), + SData: chunk.Data(), + } + spanName += ".syncing" + } else { + msg = &ChunkDeliveryMsgRetrieval{ + Addr: chunk.Address(), + SData: chunk.Data(), + } + spanName += ".retrieval" + } ctx, sp = spancontext.StartSpan( ctx, - "send.chunk.delivery") + spanName) defer sp.Finish() - msg := &ChunkDeliveryMsg{ - Addr: chunk.Address(), - SData: chunk.Data(), - } return p.SendPriority(ctx, msg, priority) } @@ -161,7 +183,7 @@ func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error { "send.offered.hashes") defer sp.Finish() - hashes, from, to, proof, err := s.SetNextBatch(f, t) + hashes, from, to, proof, err := s.setNextBatch(f, t) if err != nil { return err } @@ -204,10 +226,20 @@ func (p *Peer) setServer(s Stream, o Server, priority uint8) (*server, error) { if p.servers[s] != nil { return nil, fmt.Errorf("server %s already registered", s) } + + if p.streamer.maxPeerServers > 0 && len(p.servers) >= p.streamer.maxPeerServers { + return nil, ErrMaxPeerServers + } + + sessionIndex, err := o.SessionIndex() + if err != nil { + return nil, err + } os := &server{ - Server: o, - stream: s, - priority: priority, + Server: o, + stream: s, + priority: priority, + sessionIndex: sessionIndex, } p.servers[s] = os return os, nil @@ -346,6 +378,7 @@ func (p *Peer) removeClient(s Stream) error { return newNotFoundError("client", s) } client.close() + delete(p.clients, s) return nil } diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 6498f599dc..5ea0b1511b 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -104,8 +104,47 @@ func TestRetrieval(t *testing.T) { } } -/* +var retrievalSimServiceMap = map[string]simulation.ServiceFunc{ + "streamer": retrievalStreamerFunc, +} +func retrievalStreamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { + n := ctx.Config.Node() + addr := network.NewAddr(n) + store, datadir, err := createTestLocalStorageForID(n.ID(), addr) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyStore, store) + + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New + + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalEnabled, + Syncing: SyncingAutoSubscribe, + SyncUpdateDelay: 3 * time.Second, + }, nil) + + fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) + bucket.Store(bucketKeyFileStore, fileStore) + + cleanup = func() { + os.RemoveAll(datadir) + netStore.Close() + r.Close() + } + + return r, cleanup, nil +} + +/* The test loads a snapshot file to construct the swarm network, assuming that the snapshot file identifies a healthy kademlia network. Nevertheless a health check runs in the @@ -114,43 +153,7 @@ simulation's `action` function. The snapshot should have 'streamer' in its service list. */ func runFileRetrievalTest(nodeCount int) error { - sim := simulation.New(map[string]simulation.ServiceFunc{ - "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { - node := ctx.Config.Node() - addr := network.NewAddr(node) - store, datadir, err := createTestLocalStorageForID(node.ID(), addr) - if err != nil { - return nil, nil, err - } - bucket.Store(bucketKeyStore, store) - - localStore := store.(*storage.LocalStore) - netStore, err := storage.NewNetStore(localStore, nil) - if err != nil { - return nil, nil, err - } - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, netStore) - netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, - SyncUpdateDelay: 3 * time.Second, - }) - - fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) - bucket.Store(bucketKeyFileStore, fileStore) - - cleanup = func() { - os.RemoveAll(datadir) - netStore.Close() - r.Close() - } - - return r, cleanup, nil - - }, - }) + sim := simulation.New(retrievalSimServiceMap) defer sim.Close() log.Info("Initializing test config") @@ -200,49 +203,29 @@ func runFileRetrievalTest(nodeCount int) error { // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. - allSuccess := false - for !allSuccess { + REPEAT: + for { for _, id := range nodeIDs { - //for each expected chunk, check if it is in the local store - localChunks := conf.idToChunksMap[id] - localSuccess := true - for _, ch := range localChunks { - //get the real chunk by the index in the index array - chunk := conf.hashes[ch] - log.Trace(fmt.Sprintf("node has chunk: %s:", chunk)) - //check if the expected chunk is indeed in the localstore - var err error - //check on the node's FileStore (netstore) - item, ok := sim.NodeItem(id, bucketKeyFileStore) - if !ok { - return fmt.Errorf("No registry") - } - fileStore := item.(*storage.FileStore) - //check all chunks - for i, hash := range conf.hashes { - reader, _ := fileStore.Retrieve(context.TODO(), hash) - //check that we can read the file size and that it corresponds to the generated file size - if s, err := reader.Size(ctx, nil); err != nil || s != int64(len(randomFiles[i])) { - allSuccess = false - log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id) - } else { - log.Debug(fmt.Sprintf("File with root hash %x successfully retrieved", hash)) - } - } - if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) - localSuccess = false - } else { - log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) - } + //for each expected file, check if it is in the local store + item, ok := sim.NodeItem(id, bucketKeyFileStore) + if !ok { + return fmt.Errorf("No filestore") + } + fileStore := item.(*storage.FileStore) + //check all chunks + for i, hash := range conf.hashes { + reader, _ := fileStore.Retrieve(context.TODO(), hash) + //check that we can read the file size and that it corresponds to the generated file size + if s, err := reader.Size(ctx, nil); err != nil || s != int64(len(randomFiles[i])) { + log.Debug("Retrieve error", "err", err, "hash", hash, "nodeId", id) + time.Sleep(500 * time.Millisecond) + continue REPEAT + } + log.Debug(fmt.Sprintf("File with root hash %x successfully retrieved", hash)) } - allSuccess = localSuccess } + return nil } - if !allSuccess { - return fmt.Errorf("Not all chunks succeeded!") - } - return nil }) if result.Error != nil { @@ -263,44 +246,7 @@ simulation's `action` function. The snapshot should have 'streamer' in its service list. */ func runRetrievalTest(chunkCount int, nodeCount int) error { - sim := simulation.New(map[string]simulation.ServiceFunc{ - "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { - node := ctx.Config.Node() - addr := network.NewAddr(node) - store, datadir, err := createTestLocalStorageForID(node.ID(), addr) - if err != nil { - return nil, nil, err - } - bucket.Store(bucketKeyStore, store) - - localStore := store.(*storage.LocalStore) - netStore, err := storage.NewNetStore(localStore, nil) - if err != nil { - return nil, nil, err - } - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, netStore) - netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New - - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, - SyncUpdateDelay: 0, - }) - - fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) - bucketKeyFileStore = simulation.BucketKey("filestore") - bucket.Store(bucketKeyFileStore, fileStore) - - cleanup = func() { - os.RemoveAll(datadir) - netStore.Close() - r.Close() - } - - return r, cleanup, nil - - }, - }) + sim := simulation.New(retrievalSimServiceMap) defer sim.Close() conf := &synctestConfig{} @@ -330,8 +276,6 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { conf.addrToIDMap[string(a)] = n } - //an array for the random files - var randomFiles []string //this is the node selected for upload node := sim.RandomUpNode() item, ok := sim.NodeItem(node.ID, bucketKeyStore) @@ -349,49 +293,31 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. - allSuccess := false - for !allSuccess { + REPEAT: + for { for _, id := range nodeIDs { //for each expected chunk, check if it is in the local store - localChunks := conf.idToChunksMap[id] - localSuccess := true - for _, ch := range localChunks { - //get the real chunk by the index in the index array - chunk := conf.hashes[ch] - log.Trace(fmt.Sprintf("node has chunk: %s:", chunk)) - //check if the expected chunk is indeed in the localstore - var err error - //check on the node's FileStore (netstore) - item, ok := sim.NodeItem(id, bucketKeyFileStore) - if !ok { - return fmt.Errorf("No registry") - } - fileStore := item.(*storage.FileStore) - //check all chunks - for i, hash := range conf.hashes { - reader, _ := fileStore.Retrieve(context.TODO(), hash) - //check that we can read the file size and that it corresponds to the generated file size - if s, err := reader.Size(ctx, nil); err != nil || s != int64(len(randomFiles[i])) { - allSuccess = false - log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id) - } else { - log.Debug(fmt.Sprintf("File with root hash %x successfully retrieved", hash)) - } - } - if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) - localSuccess = false - } else { - log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) - } + //check on the node's FileStore (netstore) + item, ok := sim.NodeItem(id, bucketKeyFileStore) + if !ok { + return fmt.Errorf("No filestore") + } + fileStore := item.(*storage.FileStore) + //check all chunks + for _, hash := range conf.hashes { + reader, _ := fileStore.Retrieve(context.TODO(), hash) + //check that we can read the chunk size and that it corresponds to the generated chunk size + if s, err := reader.Size(ctx, nil); err != nil || s != int64(chunkSize) { + log.Debug("Retrieve error", "err", err, "hash", hash, "nodeId", id, "size", s) + time.Sleep(500 * time.Millisecond) + continue REPEAT + } + log.Debug(fmt.Sprintf("Chunk with root hash %x successfully retrieved", hash)) } - allSuccess = localSuccess } + // all nodes and files found, exit loop and return without error + return nil } - if !allSuccess { - return fmt.Errorf("Not all chunks succeeded!") - } - return nil }) if result.Error != nil { diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index d93afce1b1..6b92c32ae9 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -17,9 +17,7 @@ package stream import ( "context" - crand "crypto/rand" "fmt" - "io" "os" "runtime" "sync" @@ -29,8 +27,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" @@ -38,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" ) const MaxTimeout = 600 @@ -50,6 +49,17 @@ type synctestConfig struct { addrToIDMap map[string]enode.ID } +const ( + // EventTypeNode is the type of event emitted when a node is either + // created, started or stopped + EventTypeChunkCreated simulations.EventType = "chunkCreated" + EventTypeChunkOffered simulations.EventType = "chunkOffered" + EventTypeChunkWanted simulations.EventType = "chunkWanted" + EventTypeChunkDelivered simulations.EventType = "chunkDelivered" + EventTypeChunkArrived simulations.EventType = "chunkArrived" + EventTypeSimTerminated simulations.EventType = "simTerminated" +) + // Tests in this file should not request chunks from peers. // This function will panic indicating that there is a problem if request has been made. func dummyRequestFromPeers(_ context.Context, req *network.Request) (*enode.ID, chan struct{}, error) { @@ -131,41 +141,47 @@ func TestSyncingViaDirectSubscribe(t *testing.T) { } } +var simServiceMap = map[string]simulation.ServiceFunc{ + "streamer": streamerFunc, +} + +func streamerFunc(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { + n := ctx.Config.Node() + addr := network.NewAddr(n) + store, datadir, err := createTestLocalStorageForID(n.ID(), addr) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyStore, store) + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New + + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, + SyncUpdateDelay: 3 * time.Second, + }, nil) + + bucket.Store(bucketKeyRegistry, r) + + cleanup = func() { + os.RemoveAll(datadir) + netStore.Close() + r.Close() + } + + return r, cleanup, nil + +} + func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { - sim := simulation.New(map[string]simulation.ServiceFunc{ - "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { - n := ctx.Config.Node() - addr := network.NewAddr(n) - store, datadir, err := createTestLocalStorageForID(n.ID(), addr) - if err != nil { - return nil, nil, err - } - bucket.Store(bucketKeyStore, store) - localStore := store.(*storage.LocalStore) - netStore, err := storage.NewNetStore(localStore, nil) - if err != nil { - return nil, nil, err - } - kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - delivery := NewDelivery(kad, netStore) - netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New - - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ - DoSync: true, - SyncUpdateDelay: 3 * time.Second, - }) - bucket.Store(bucketKeyRegistry, r) - - cleanup = func() { - os.RemoveAll(datadir) - netStore.Close() - r.Close() - } - - return r, cleanup, nil - - }, - }) + sim := simulation.New(simServiceMap) defer sim.Close() log.Info("Initializing test config") @@ -193,18 +209,28 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal("unexpected disconnect") cancelSimRun() } }() - result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + result := runSim(conf, ctx, sim, chunkCount) + + if result.Error != nil { + t.Fatal(result.Error) + } + log.Info("Simulation ended") +} + +func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulation, chunkCount int) simulation.Result { + + return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() for _, n := range nodeIDs { //get the kademlia overlay address from this ID @@ -229,12 +255,19 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { if err != nil { return err } + for _, h := range hashes { + evt := &simulations.Event{ + Type: EventTypeChunkCreated, + Node: sim.Net.GetNode(node.ID), + Data: h.String(), + } + sim.Net.Events().Send(evt) + } conf.hashes = append(conf.hashes, hashes...) mapKeysToNodes(conf) // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. - allSuccess := false var gDir string var globalStore *mockdb.GlobalStore if *useMockStore { @@ -250,12 +283,11 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { } }() } - for !allSuccess { - allSuccess = true + REPEAT: + for { for _, id := range nodeIDs { //for each expected chunk, check if it is in the local store localChunks := conf.idToChunksMap[id] - localSuccess := true for _, ch := range localChunks { //get the real chunk by the index in the index array chunk := conf.hashes[ch] @@ -276,30 +308,23 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { _, err = lstore.Get(ctx, chunk) } if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) - localSuccess = false + log.Debug(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) // Do not get crazy with logging the warn message time.Sleep(500 * time.Millisecond) - } else { - log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) + continue REPEAT } - } - if !localSuccess { - allSuccess = false - break + evt := &simulations.Event{ + Type: EventTypeChunkArrived, + Node: sim.Net.GetNode(id), + Data: chunk.String(), + } + sim.Net.Events().Send(evt) + log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) } } + return nil } - if !allSuccess { - return fmt.Errorf("Not all chunks succeeded!") - } - return nil }) - - if result.Error != nil { - t.Fatal(result.Error) - } - log.Info("Simulation ended") } /* @@ -332,7 +357,10 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) delivery := NewDelivery(kad, netStore) netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), nil) + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingRegisterOnly, + }, nil) bucket.Store(bucketKeyRegistry, r) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) @@ -373,12 +401,12 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal("unexpected disconnect") cancelSimRun() } @@ -399,7 +427,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) var subscriptionCount int - filter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(4) + filter := simulation.NewPeerEventsFilter().ReceivedMessages().Protocol("stream").MsgCode(4) eventC := sim.PeerEvents(ctx, nodeIDs, filter) for j, node := range nodeIDs { @@ -459,13 +487,11 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) } // File retrieval check is repeated until all uploaded files are retrieved from all nodes // or until the timeout is reached. - allSuccess := false - for !allSuccess { - allSuccess = true + REPEAT: + for { for _, id := range nodeIDs { //for each expected chunk, check if it is in the local store localChunks := conf.idToChunksMap[id] - localSuccess := true for _, ch := range localChunks { //get the real chunk by the index in the index array chunk := conf.hashes[ch] @@ -486,24 +512,16 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) _, err = lstore.Get(ctx, chunk) } if err != nil { - log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) - localSuccess = false + log.Debug(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id)) // Do not get crazy with logging the warn message time.Sleep(500 * time.Millisecond) - } else { - log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) + continue REPEAT } - } - if !localSuccess { - allSuccess = false - break + log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id)) } } + return nil } - if !allSuccess { - return fmt.Errorf("Not all chunks succeeded!") - } - return nil }) if result.Error != nil { @@ -583,7 +601,7 @@ func uploadFileToSingleNodeStore(id enode.ID, chunkCount int, lstore *storage.Lo size := chunkSize var rootAddrs []storage.Address for i := 0; i < chunkCount; i++ { - rk, wait, err := fileStore.Store(context.TODO(), io.LimitReader(crand.Reader, int64(size)), int64(size), false) + rk, wait, err := fileStore.Store(context.TODO(), testutil.RandomReader(i, size), int64(size), false) if err != nil { return nil, err } diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index ea7cce8cb4..32e107823b 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -18,8 +18,10 @@ package stream import ( "context" + "errors" "fmt" "math" + "reflect" "sync" "time" @@ -46,6 +48,31 @@ const ( HashSize = 32 ) +//Enumerate options for syncing and retrieval +type SyncingOption int +type RetrievalOption int + +//Syncing options +const ( + //Syncing disabled + SyncingDisabled SyncingOption = iota + //Register the client and the server but not subscribe + SyncingRegisterOnly + //Both client and server funcs are registered, subscribe sent automatically + SyncingAutoSubscribe +) + +const ( + //Retrieval disabled. Used mostly for tests to isolate syncing features (i.e. syncing only) + RetrievalDisabled RetrievalOption = iota + //Only the client side of the retrieve request is registered. + //(light nodes do not serve retrieve requests) + //once the client is registered, subscription to retrieve request stream is always sent + RetrievalClientOnly + //Both client and server funcs are registered, subscribe sent automatically + RetrievalEnabled +) + // Registry registry for outgoing and incoming streamer constructors type Registry struct { addr enode.ID @@ -59,25 +86,33 @@ type Registry struct { peers map[enode.ID]*Peer delivery *Delivery intervalsStore state.Store - doRetrieve bool + autoRetrieval bool //automatically subscribe to retrieve request stream + maxPeerServers int + spec *protocols.Spec //this protocol's spec + balance protocols.Balance //implements protocols.Balance, for accounting + prices protocols.Prices //implements protocols.Prices, provides prices to accounting } // RegistryOptions holds optional values for NewRegistry constructor. type RegistryOptions struct { SkipCheck bool - DoSync bool - DoRetrieve bool + Syncing SyncingOption //Defines syncing behavior + Retrieval RetrievalOption //Defines retrieval behavior SyncUpdateDelay time.Duration + MaxPeerServers int // The limit of servers for each peer in registry } // NewRegistry is Streamer constructor -func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry { +func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions, balance protocols.Balance) *Registry { if options == nil { options = &RegistryOptions{} } if options.SyncUpdateDelay <= 0 { options.SyncUpdateDelay = 15 * time.Second } + //check if retriaval has been disabled + retrieval := options.Retrieval != RetrievalDisabled + streamer := &Registry{ addr: localID, skipCheck: options.SkipCheck, @@ -86,20 +121,40 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy peers: make(map[enode.ID]*Peer), delivery: delivery, intervalsStore: intervalsStore, - doRetrieve: options.DoRetrieve, + autoRetrieval: retrieval, + maxPeerServers: options.MaxPeerServers, + balance: balance, } + streamer.setupSpec() + streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer - streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { - return NewSwarmChunkServer(delivery.chunkStore), nil - }) - streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { - return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) - }) - RegisterSwarmSyncerServer(streamer, syncChunkStore) - RegisterSwarmSyncerClient(streamer, syncChunkStore) - if options.DoSync { + //if retrieval is enabled, register the server func, so that retrieve requests will be served (non-light nodes only) + if options.Retrieval == RetrievalEnabled { + streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) { + if !live { + return nil, errors.New("only live retrieval requests supported") + } + return NewSwarmChunkServer(delivery.chunkStore), nil + }) + } + + //if retrieval is not disabled, register the client func (both light nodes and normal nodes can issue retrieve requests) + if options.Retrieval != RetrievalDisabled { + streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { + return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) + }) + } + + //If syncing is not disabled, the syncing functions are registered (both client and server) + if options.Syncing != SyncingDisabled { + RegisterSwarmSyncerServer(streamer, syncChunkStore) + RegisterSwarmSyncerClient(streamer, syncChunkStore) + } + + //if syncing is set to automatically subscribe to the syncing stream, start the subscription process + if options.Syncing == SyncingAutoSubscribe { // latestIntC function ensures that // - receiving from the in chan is not blocked by processing inside the for loop // - the latest int value is delivered to the loop after the processing is done @@ -180,6 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy return streamer } +//we need to construct a spec instance per node instance +func (r *Registry) setupSpec() { + //first create the "bare" spec + r.createSpec() + //if balance is nil, this node has been started without swap support (swapEnabled flag is false) + if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() { + //swap is enabled, so setup the hook + r.spec.Hook = protocols.NewAccounting(r.balance, r.prices) + } +} + // RegisterClient registers an incoming streamer constructor func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) { r.clientMu.Lock() @@ -268,7 +334,6 @@ func (r *Registry) Subscribe(peerId enode.ID, s Stream, h *Range, priority uint8 if err != nil { return err } - if s.Live && h != nil { if err := peer.setClientParams( getHistoryStream(s), @@ -371,8 +436,8 @@ func (r *Registry) Run(p *network.BzzPeer) error { defer close(sp.quit) defer sp.close() - if r.doRetrieve { - err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", false), nil, Top) + if r.autoRetrieval && !p.LightNode { + err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", true), nil, Top) if err != nil { return err } @@ -445,7 +510,7 @@ func (r *Registry) updateSyncing() { } func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := protocols.NewPeer(p, rw, Spec) + peer := protocols.NewPeer(p, rw, r.spec) bp := network.NewBzzPeer(peer) np := network.NewPeer(bp, r.delivery.kad) r.delivery.kad.On(np) @@ -475,8 +540,13 @@ func (p *Peer) HandleMsg(ctx context.Context, msg interface{}) error { case *WantedHashesMsg: return p.handleWantedHashesMsg(ctx, msg) - case *ChunkDeliveryMsg: - return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, msg) + case *ChunkDeliveryMsgRetrieval: + //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) + + case *ChunkDeliveryMsgSyncing: + //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) case *RetrieveRequestMsg: return p.streamer.delivery.handleRetrieveRequestMsg(ctx, p, msg) @@ -497,10 +567,38 @@ type server struct { stream Stream priority uint8 currentBatch []byte + sessionIndex uint64 +} + +// setNextBatch adjusts passed interval based on session index and whether +// stream is live or history. It calls Server SetNextBatch with adjusted +// interval and returns batch hashes and their interval. +func (s *server) setNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { + if s.stream.Live { + if from == 0 { + from = s.sessionIndex + } + if to <= from || from >= s.sessionIndex { + to = math.MaxUint64 + } + } else { + if (to < from && to != 0) || from > s.sessionIndex { + return nil, 0, 0, nil, nil + } + if to == 0 || to > s.sessionIndex { + to = s.sessionIndex + } + } + return s.SetNextBatch(from, to) } // Server interface for outgoing peer Streamer type Server interface { + // SessionIndex is called when a server is initialized + // to get the current cursor state of the stream data. + // Based on this index, live and history stream intervals + // will be adjusted before calling SetNextBatch. + SessionIndex() (uint64, error) SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) GetData(context.Context, []byte) ([]byte, error) Close() @@ -636,34 +734,43 @@ func (c *clientParams) clientCreated() { close(c.clientCreatedC) } -// Spec is the spec of the streamer protocol -var Spec = &protocols.Spec{ - Name: "stream", - Version: 6, - MaxMsgSize: 10 * 1024 * 1024, - Messages: []interface{}{ - UnsubscribeMsg{}, - OfferedHashesMsg{}, - WantedHashesMsg{}, - TakeoverProofMsg{}, - SubscribeMsg{}, - RetrieveRequestMsg{}, - ChunkDeliveryMsg{}, - SubscribeErrorMsg{}, - RequestSubscriptionMsg{}, - QuitMsg{}, - }, +//GetSpec returns the streamer spec to callers +//This used to be a global variable but for simulations with +//multiple nodes its fields (notably the Hook) would be overwritten +func (r *Registry) GetSpec() *protocols.Spec { + return r.spec +} + +func (r *Registry) createSpec() { + // Spec is the spec of the streamer protocol + var spec = &protocols.Spec{ + Name: "stream", + Version: 8, + MaxMsgSize: 10 * 1024 * 1024, + Messages: []interface{}{ + UnsubscribeMsg{}, + OfferedHashesMsg{}, + WantedHashesMsg{}, + TakeoverProofMsg{}, + SubscribeMsg{}, + RetrieveRequestMsg{}, + ChunkDeliveryMsgRetrieval{}, + SubscribeErrorMsg{}, + RequestSubscriptionMsg{}, + QuitMsg{}, + ChunkDeliveryMsgSyncing{}, + }, + } + r.spec = spec } func (r *Registry) Protocols() []p2p.Protocol { return []p2p.Protocol{ { - Name: Spec.Name, - Version: Spec.Version, - Length: Spec.Length(), + Name: r.spec.Name, + Version: r.spec.Version, + Length: r.spec.Length(), Run: r.runProtocol, - // NodeInfo: , - // PeerInfo: , }, } } diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index e13cc8c29f..16c74d3b34 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -19,6 +19,8 @@ package stream import ( "bytes" "context" + "errors" + "strconv" "testing" "time" @@ -27,7 +29,7 @@ import ( ) func TestStreamerSubscribe(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -41,7 +43,7 @@ func TestStreamerSubscribe(t *testing.T) { } func TestStreamerRequestSubscription(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -55,11 +57,12 @@ func TestStreamerRequestSubscription(t *testing.T) { } var ( - hash0 = sha3.Sum256([]byte{0}) - hash1 = sha3.Sum256([]byte{1}) - hash2 = sha3.Sum256([]byte{2}) - hashesTmp = append(hash0[:], hash1[:]...) - hashes = append(hashesTmp, hash2[:]...) + hash0 = sha3.Sum256([]byte{0}) + hash1 = sha3.Sum256([]byte{1}) + hash2 = sha3.Sum256([]byte{2}) + hashesTmp = append(hash0[:], hash1[:]...) + hashes = append(hashesTmp, hash2[:]...) + corruptHashes = append(hashes[:40]) ) type testClient struct { @@ -104,15 +107,21 @@ func (self *testClient) BatchDone(Stream, uint64, []byte, []byte) func() (*Takeo func (self *testClient) Close() {} type testServer struct { - t string + t string + sessionIndex uint64 } -func newTestServer(t string) *testServer { +func newTestServer(t string, sessionIndex uint64) *testServer { return &testServer{ - t: t, + t: t, + sessionIndex: sessionIndex, } } +func (s *testServer) SessionIndex() (uint64, error) { + return s.sessionIndex, nil +} + func (self *testServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { return make([]byte, HashSize), from + 1, to + 1, nil, nil } @@ -125,7 +134,7 @@ func (self *testServer) Close() { } func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -218,7 +227,7 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -227,7 +236,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { stream := NewStream("foo", "", false) streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 10), nil }) node := tester.Nodes[0] @@ -285,7 +294,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -294,7 +303,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { stream := NewStream("foo", "", true) streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 0), nil }) node := tester.Nodes[0] @@ -321,7 +330,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { }, Hashes: make([]byte, HashSize), From: 1, - To: 1, + To: 0, }, Peer: node.ID(), }, @@ -351,14 +360,14 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { } func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) } streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 0), nil }) stream := NewStream("bar", "", true) @@ -395,7 +404,7 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -404,9 +413,7 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { stream := NewStream("foo", "", true) streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return &testServer{ - t: t, - }, nil + return newTestServer(t, 10), nil }) node := tester.Nodes[0] @@ -445,8 +452,8 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { HandoverProof: &HandoverProof{ Handover: &Handover{}, }, - From: 1, - To: 1, + From: 11, + To: 0, Hashes: make([]byte, HashSize), }, Peer: node.ID(), @@ -459,8 +466,73 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { } } +func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) { + tester, streamer, _, teardown, err := newStreamerTester(t, nil) + defer teardown() + if err != nil { + t.Fatal(err) + } + + stream := NewStream("foo", "", true) + + var tc *testClient + + streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) { + tc = newTestClient(t) + return tc, nil + }) + + node := tester.Nodes[0] + + err = streamer.Subscribe(node.ID(), stream, NewRange(5, 8), Top) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + History: NewRange(5, 8), + Priority: Top, + }, + Peer: node.ID(), + }, + }, + }, + p2ptest.Exchange{ + Label: "Corrupt offered hash message", + Triggers: []p2ptest.Trigger{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: corruptHashes, + From: 5, + To: 8, + Stream: stream, + }, + Peer: node.ID(), + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + expectedError := errors.New("Message handler error: (msg code 1): error invalid hashes length (len: 40)") + if err := tester.TestDisconnected(&p2ptest.Disconnect{Peer: node.ID(), Error: expectedError}); err != nil { + t.Fatal(err) + } +} + func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) @@ -559,14 +631,14 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { } func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t, nil) defer teardown() if err != nil { t.Fatal(err) } streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { - return newTestServer(t), nil + return newTestServer(t, 10), nil }) node := tester.Nodes[0] @@ -626,8 +698,8 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { HandoverProof: &HandoverProof{ Handover: &Handover{}, }, - From: 1, - To: 1, + From: 11, + To: 0, Hashes: make([]byte, HashSize), }, Peer: node.ID(), @@ -685,3 +757,167 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { t.Fatal(err) } } + +// TestMaxPeerServersWithUnsubscribe creates a registry with a limited +// number of stream servers, and performs a test with subscriptions and +// unsubscriptions, checking if unsubscriptions will remove streams, +// leaving place for new streams. +func TestMaxPeerServersWithUnsubscribe(t *testing.T) { + var maxPeerServers = 6 + tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, + MaxPeerServers: maxPeerServers, + }) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { + return newTestServer(t, 0), nil + }) + + node := tester.Nodes[0] + + for i := 0; i < maxPeerServers+10; i++ { + stream := NewStream("foo", strconv.Itoa(i), true) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + Stream: stream, + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: make([]byte, HashSize), + From: 1, + To: 0, + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "unsubscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &UnsubscribeMsg{ + Stream: stream, + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + } +} + +// TestMaxPeerServersWithoutUnsubscribe creates a registry with a limited +// number of stream servers, and performs subscriptions to detect subscriptions +// error message exchange. +func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { + var maxPeerServers = 6 + tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + MaxPeerServers: maxPeerServers, + }) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { + return newTestServer(t, 0), nil + }) + + node := tester.Nodes[0] + + for i := 0; i < maxPeerServers+10; i++ { + stream := NewStream("foo", strconv.Itoa(i), true) + + if i >= maxPeerServers { + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 7, + Msg: &SubscribeErrorMsg{ + Error: ErrMaxPeerServers.Error(), + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + continue + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Triggers: []p2ptest.Trigger{ + { + Code: 4, + Msg: &SubscribeMsg{ + Stream: stream, + Priority: Top, + }, + Peer: node.ID(), + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &OfferedHashesMsg{ + Stream: stream, + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: make([]byte, HashSize), + From: 1, + To: 0, + }, + Peer: node.ID(), + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + } +} diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index 38b3078d2f..4bfbac8b0d 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -18,7 +18,6 @@ package stream import ( "context" - "math" "strconv" "time" @@ -36,38 +35,27 @@ const ( // * live request delivery with or without checkback // * (live/non-live historical) chunk syncing per proximity bin type SwarmSyncerServer struct { - po uint8 - store storage.SyncChunkStore - sessionAt uint64 - start uint64 - live bool - quit chan struct{} + po uint8 + store storage.SyncChunkStore + quit chan struct{} } -// NewSwarmSyncerServer is contructor for SwarmSyncerServer -func NewSwarmSyncerServer(live bool, po uint8, syncChunkStore storage.SyncChunkStore) (*SwarmSyncerServer, error) { - sessionAt := syncChunkStore.BinIndex(po) - var start uint64 - if live { - start = sessionAt - } +// NewSwarmSyncerServer is constructor for SwarmSyncerServer +func NewSwarmSyncerServer(po uint8, syncChunkStore storage.SyncChunkStore) (*SwarmSyncerServer, error) { return &SwarmSyncerServer{ - po: po, - store: syncChunkStore, - sessionAt: sessionAt, - start: start, - live: live, - quit: make(chan struct{}), + po: po, + store: syncChunkStore, + quit: make(chan struct{}), }, nil } func RegisterSwarmSyncerServer(streamer *Registry, syncChunkStore storage.SyncChunkStore) { - streamer.RegisterServerFunc("SYNC", func(p *Peer, t string, live bool) (Server, error) { + streamer.RegisterServerFunc("SYNC", func(_ *Peer, t string, _ bool) (Server, error) { po, err := ParseSyncBinKey(t) if err != nil { return nil, err } - return NewSwarmSyncerServer(live, po, syncChunkStore) + return NewSwarmSyncerServer(po, syncChunkStore) }) // streamer.RegisterServerFunc(stream, func(p *Peer) (Server, error) { // return NewOutgoingProvableSwarmSyncer(po, db) @@ -88,25 +76,15 @@ func (s *SwarmSyncerServer) GetData(ctx context.Context, key []byte) ([]byte, er return chunk.Data(), nil } +// SessionIndex returns current storage bin (po) index. +func (s *SwarmSyncerServer) SessionIndex() (uint64, error) { + return s.store.BinIndex(s.po), nil +} + // GetBatch retrieves the next batch of hashes from the dbstore func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { var batch []byte i := 0 - if s.live { - if from == 0 { - from = s.start - } - if to <= from || from >= s.sessionAt { - to = math.MaxUint64 - } - } else { - if (to < from && to != 0) || from > s.sessionAt { - return nil, 0, 0, nil, nil - } - if to == 0 || to > s.sessionAt { - to = s.sessionAt - } - } var ticker *time.Ticker defer func() { diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index f2be3bef99..fe20bab266 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -18,9 +18,7 @@ package stream import ( "context" - crand "crypto/rand" "fmt" - "io" "io/ioutil" "math" "os" @@ -30,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/log" @@ -39,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/testutil" ) const dataChunkCount = 200 @@ -62,6 +60,9 @@ func createMockStore(globalStore *mockdb.GlobalStore, id enode.ID, addr *network params.Init(datadir) params.BaseKey = addr.Over() lstore, err = storage.NewLocalStore(params, mockStore) + if err != nil { + return nil, "", err + } return lstore, datadir, nil } @@ -114,8 +115,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bucket.Store(bucketKeyDelivery, delivery) r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, SkipCheck: skipCheck, - }) + }, nil) fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) bucket.Store(bucketKeyFileStore, fileStore) @@ -147,13 +150,13 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) t.Fatal(d.Error) } } @@ -178,7 +181,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } fileStore := item.(*storage.FileStore) size := chunkCount * chunkSize - _, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false) + _, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false) if err != nil { t.Fatal(err.Error()) } diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go new file mode 100644 index 0000000000..437c17e5e2 --- /dev/null +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -0,0 +1,225 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build withserver + +package stream + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/network/simulation" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +/* +The tests in this file need to be executed with + + -tags=withserver + +Also, they will stall if executed stand-alone, because they wait +for the visualization frontend to send a POST /runsim message. +*/ + +//setup the sim, evaluate nodeCount and chunkCount and create the sim +func setupSim(serviceMap map[string]simulation.ServiceFunc) (int, int, *simulation.Simulation) { + nodeCount := *nodes + chunkCount := *chunks + + if nodeCount == 0 || chunkCount == 0 { + nodeCount = 32 + chunkCount = 1 + } + + //setup the simulation with server, which means the sim won't run + //until it receives a POST /runsim from the frontend + sim := simulation.New(serviceMap).WithServer(":8888") + return nodeCount, chunkCount, sim +} + +//watch for disconnections and wait for healthy +func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) { + ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) + + if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + panic(err) + } + + disconnections := sim.PeerEvents( + context.Background(), + sim.NodeIDs(), + simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + ) + + go func() { + for d := range disconnections { + log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + panic("unexpected disconnect") + cancelSimRun() + } + }() + + return ctx, cancelSimRun +} + +//This test requests bogus hashes into the network +func TestNonExistingHashesWithServer(t *testing.T) { + nodeCount, _, sim := setupSim(retrievalSimServiceMap) + defer sim.Close() + + err := sim.UploadSnapshot(fmt.Sprintf("testing/snapshot_%d.json", nodeCount)) + if err != nil { + panic(err) + } + + ctx, cancelSimRun := watchSim(sim) + defer cancelSimRun() + + //in order to get some meaningful visualization, it is beneficial + //to define a minimum duration of this test + testDuration := 20 * time.Second + + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + //check on the node's FileStore (netstore) + id := sim.RandomUpNode().ID + item, ok := sim.NodeItem(id, bucketKeyFileStore) + if !ok { + t.Fatalf("No filestore") + } + fileStore := item.(*storage.FileStore) + //create a bogus hash + fakeHash := storage.GenerateRandomChunk(1000).Address() + //try to retrieve it - will propagate RetrieveRequestMsg into the network + reader, _ := fileStore.Retrieve(context.TODO(), fakeHash) + if _, err := reader.Size(ctx, nil); err != nil { + log.Debug("expected error for non-existing chunk") + } + //sleep so that the frontend can have something to display + time.Sleep(testDuration) + + return nil + }) + if result.Error != nil { + sendSimTerminatedEvent(sim) + t.Fatal(result.Error) + } + + sendSimTerminatedEvent(sim) + +} + +//send a termination event to the frontend +func sendSimTerminatedEvent(sim *simulation.Simulation) { + evt := &simulations.Event{ + Type: EventTypeSimTerminated, + Control: false, + } + sim.Net.Events().Send(evt) +} + +//This test is the same as the snapshot sync test, +//but with a HTTP server +//It also sends some custom events so that the frontend +//can visualize messages like SendOfferedMsg, WantedHashesMsg, DeliveryMsg +func TestSnapshotSyncWithServer(t *testing.T) { + + nodeCount, chunkCount, sim := setupSim(simServiceMap) + defer sim.Close() + + log.Info("Initializing test config") + + conf := &synctestConfig{} + //map of discover ID to indexes of chunks expected at that ID + conf.idToChunksMap = make(map[discover.NodeID][]int) + //map of overlay address to discover ID + conf.addrToIDMap = make(map[string]discover.NodeID) + //array where the generated chunk hashes will be stored + conf.hashes = make([]storage.Address, 0) + + err := sim.UploadSnapshot(fmt.Sprintf("testing/snapshot_%d.json", nodeCount)) + if err != nil { + panic(err) + } + + ctx, cancelSimRun := watchSim(sim) + defer cancelSimRun() + + //setup filters in the event feed + offeredHashesFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(1) + wantedFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(2) + deliveryFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(6) + eventC := sim.PeerEvents(ctx, sim.UpNodeIDs(), offeredHashesFilter, wantedFilter, deliveryFilter) + + quit := make(chan struct{}) + + go func() { + for e := range eventC { + select { + case <-quit: + fmt.Println("quitting event loop") + return + default: + } + if e.Error != nil { + t.Fatal(e.Error) + } + if *e.Event.MsgCode == uint64(1) { + evt := &simulations.Event{ + Type: EventTypeChunkOffered, + Node: sim.Net.GetNode(e.NodeID), + Control: false, + } + sim.Net.Events().Send(evt) + } else if *e.Event.MsgCode == uint64(2) { + evt := &simulations.Event{ + Type: EventTypeChunkWanted, + Node: sim.Net.GetNode(e.NodeID), + Control: false, + } + sim.Net.Events().Send(evt) + } else if *e.Event.MsgCode == uint64(6) { + evt := &simulations.Event{ + Type: EventTypeChunkDelivered, + Node: sim.Net.GetNode(e.NodeID), + Control: false, + } + sim.Net.Events().Send(evt) + } + } + }() + //run the sim + result := runSim(conf, ctx, sim, chunkCount) + + //send terminated event + evt := &simulations.Event{ + Type: EventTypeSimTerminated, + Control: false, + } + sim.Net.Events().Send(evt) + + if result.Error != nil { + panic(result.Error) + } + close(quit) + log.Info("Simulation ended") +} diff --git a/swarm/network_test.go b/swarm/network_test.go index 0b57fab333..d84f28147f 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -150,14 +150,14 @@ func TestSwarmNetwork(t *testing.T) { { name: "dec_inc_node_count", steps: []testSwarmNetworkStep{ - { - nodeCount: 5, - }, { nodeCount: 3, }, { - nodeCount: 10, + nodeCount: 1, + }, + { + nodeCount: 5, }, }, options: &testSwarmNetworkOptions{ @@ -335,7 +335,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() - shuffle(len(nodeIDs), func(i, j int) { + rand.Shuffle(len(nodeIDs), func(i, j int) { nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i] }) for _, id := range nodeIDs { @@ -404,7 +404,7 @@ func retrieve( nodeStatusM *sync.Map, totalFoundCount *uint64, ) (missing uint64) { - shuffle(len(files), func(i, j int) { + rand.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) @@ -499,32 +499,3 @@ func retrieve( return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount) } - -// Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333 -// -// Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped. -// -// shuffle pseudo-randomizes the order of elements. -// n is the number of elements. Shuffle panics if n < 0. -// swap swaps the elements with indexes i and j. -func shuffle(n int, swap func(i, j int)) { - if n < 0 { - panic("invalid argument to Shuffle") - } - - // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle - // Shuffle really ought not be called with n that doesn't fit in 32 bits. - // Not only will it take a very long time, but with 2³¹! possible permutations, - // there's no way that any PRNG can have a big enough internal state to - // generate even a minuscule percentage of the possible permutations. - // Nevertheless, the right API signature accepts an int n, so handle it as best we can. - i := n - 1 - for ; i > 1<<31-1-1; i-- { - j := int(rand.Int63n(int64(i + 1))) - swap(i, j) - } - for ; i > 0; i-- { - j := int(rand.Int31n(int32(i + 1))) - swap(i, j) - } -} diff --git a/swarm/pot/address.go b/swarm/pot/address.go index 3974ebcaac..728dac14ee 100644 --- a/swarm/pot/address.go +++ b/swarm/pot/address.go @@ -79,46 +79,6 @@ func (a Address) Bytes() []byte { return a[:] } -/* -Proximity(x, y) returns the proximity order of the MSB distance between x and y - -The distance metric MSB(x, y) of two equal length byte sequences x an y is the -value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. -the binary cast is big endian: most significant bit first (=MSB). - -Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. -It is defined as the reverse rank of the integer part of the base 2 -logarithm of the distance. -It is calculated by counting the number of common leading zeros in the (MSB) -binary representation of the x^y. - -(0 farthest, 255 closest, 256 self) -*/ -func proximity(one, other Address) (ret int, eq bool) { - return posProximity(one, other, 0) -} - -// posProximity(a, b, pos) returns proximity order of b wrt a (symmetric) pretending -// the first pos bits match, checking only bits index >= pos -func posProximity(one, other Address, pos int) (ret int, eq bool) { - for i := pos / 8; i < len(one); i++ { - if one[i] == other[i] { - continue - } - oxo := one[i] ^ other[i] - start := 0 - if i == pos/8 { - start = pos % 8 - } - for j := start; j < 8; j++ { - if (oxo>>uint8(7-j))&0x01 != 0 { - return i*8 + j, false - } - } - } - return len(one) * 8, true -} - // ProxCmp compares the distances a->target and b->target. // Returns -1 if a is closer to target, 1 if b is closer to target // and 0 if they are equal. diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 48edc6ccef..8f2f0e8059 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -252,7 +252,13 @@ func newServices() adapters.Services { ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) + if err != nil { + return nil, err + } privkey, err := w.GetPrivateKey(keys) + if err != nil { + return nil, err + } psparams := pss.NewPssParams().WithPrivateKey(privkey) pskad := kademlia(ctx.Config.ID) ps, err := pss.NewPss(pskad, psparams) @@ -288,10 +294,6 @@ type testStore struct { values map[string][]byte } -func newTestStore() *testStore { - return &testStore{values: make(map[string][]byte)} -} - func (t *testStore) Load(key string) ([]byte, error) { return nil, nil } diff --git a/swarm/pss/notify/notify.go b/swarm/pss/notify/notify.go index 723092c32d..3731fb9dbb 100644 --- a/swarm/pss/notify/notify.go +++ b/swarm/pss/notify/notify.go @@ -317,7 +317,7 @@ func (c *Controller) handleStartMsg(msg *Msg, keyid string) (err error) { return err } - // TODO this is set to zero-length byte pending decision on protocol for initial message, whether it should include message or not, and how to trigger the initial message so that current state of MRU is sent upon subscription + // TODO this is set to zero-length byte pending decision on protocol for initial message, whether it should include message or not, and how to trigger the initial message so that current state of Swarm feed is sent upon subscription notify := []byte{} replyMsg := NewMsg(MsgCodeNotifyWithKey, msg.namestring, make([]byte, len(notify)+symKeyLength)) copy(replyMsg.Payload, notify) diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index 675b41ada2..d4d383a6ba 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -223,7 +223,13 @@ func newServices(allowRaw bool) adapters.Services { ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) + if err != nil { + return nil, err + } privkey, err := w.GetPrivateKey(keys) + if err != nil { + return nil, err + } pssp := pss.NewPssParams().WithPrivateKey(privkey) pssp.MsgTTL = time.Second * 30 pssp.AllowRaw = allowRaw diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index f4209fea59..4ef3e90a04 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -93,11 +93,17 @@ func testProtocol(t *testing.T) { lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + if err != nil { + t.Fatal(err) + } defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + if err != nil { + t.Fatal(err) + } defer rsub.Unsubscribe() // set reciprocal public keys diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 574714114b..66a90be620 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -976,11 +976,6 @@ func TestNetwork10000(t *testing.T) { } func testNetwork(t *testing.T) { - type msgnotifyC struct { - id enode.ID - msgIdx int - } - paramstring := strings.Split(t.Name(), "/") nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0) msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0) diff --git a/swarm/pss/types.go b/swarm/pss/types.go index 1e33ecdcac..56c2c51dc0 100644 --- a/swarm/pss/types.go +++ b/swarm/pss/types.go @@ -169,10 +169,6 @@ type stateStore struct { values map[string][]byte } -func newStateStore() *stateStore { - return &stateStore{values: make(map[string][]byte)} -} - func (store *stateStore) Load(key string) ([]byte, error) { return nil, nil } diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index bed2b11458..fb7d35b000 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -2,19 +2,17 @@ package sctx import "context" -type ContextKey int - -const ( - HTTPRequestIDKey ContextKey = iota - requestHostKey +type ( + HTTPRequestIDKey struct{} + requestHostKey struct{} ) func SetHost(ctx context.Context, domain string) context.Context { - return context.WithValue(ctx, requestHostKey, domain) + return context.WithValue(ctx, requestHostKey{}, domain) } func GetHost(ctx context.Context) string { - v, ok := ctx.Value(requestHostKey).(string) + v, ok := ctx.Value(requestHostKey{}).(string) if ok { return v } diff --git a/swarm/state/dbstore.go b/swarm/state/dbstore.go index 5e5c172b25..b0aa92e272 100644 --- a/swarm/state/dbstore.go +++ b/swarm/state/dbstore.go @@ -69,7 +69,7 @@ func (s *DBStore) Get(key string, i interface{}) (err error) { // Put stores an object that implements Binary for a specific key. func (s *DBStore) Put(key string, i interface{}) (err error) { - bytes := []byte{} + var bytes []byte marshaler, ok := i.(encoding.BinaryMarshaler) if !ok { diff --git a/swarm/state/dbstore_test.go b/swarm/state/dbstore_test.go index 6683e788f7..f7098956d0 100644 --- a/swarm/state/dbstore_test.go +++ b/swarm/state/dbstore_test.go @@ -112,6 +112,9 @@ func testPersistedStore(t *testing.T, store Store) { as := []string{} err = store.Get("key2", &as) + if err != nil { + t.Fatal(err) + } if len(as) != 3 { t.Fatalf("serialized array did not match expectation") diff --git a/swarm/state/inmemorystore.go b/swarm/state/inmemorystore.go index 1ca25404a1..3ba48592bc 100644 --- a/swarm/state/inmemorystore.go +++ b/swarm/state/inmemorystore.go @@ -59,7 +59,7 @@ func (s *InmemoryStore) Get(key string, i interface{}) (err error) { func (s *InmemoryStore) Put(key string, i interface{}) (err error) { s.mu.Lock() defer s.mu.Unlock() - bytes := []byte{} + var bytes []byte marshaler, ok := i.(encoding.BinaryMarshaler) if !ok { diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index db719ca047..1f847edcb9 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -19,13 +19,13 @@ package storage import ( "bytes" "context" - "crypto/rand" "encoding/binary" "fmt" "io" "testing" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) /* @@ -47,7 +47,7 @@ func newTestHasherStore(store ChunkStore, hash string) *hasherStore { } func testRandomBrokenData(n int, tester *chunkerTester) { - data := io.LimitReader(rand.Reader, int64(n)) + data := testutil.RandomReader(1, n) brokendata := brokenLimitReader(data, n, n/2) buf := make([]byte, n) @@ -56,7 +56,7 @@ func testRandomBrokenData(n int, tester *chunkerTester) { tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) } - data = io.LimitReader(rand.Reader, int64(n)) + data = testutil.RandomReader(2, n) brokendata = brokenLimitReader(data, n, n/2) putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) @@ -77,7 +77,8 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = GenerateRandomData(n) + input = testutil.RandomBytes(1, n) + data = bytes.NewReader(input) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -118,14 +119,13 @@ func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) // testing partial read for i := 1; i < n; i += 10000 { readableLength := n - i - output := make([]byte, readableLength) r, err := reader.ReadAt(output, int64(i)) if r != readableLength || err != io.EOF { tester.t.Fatalf("readAt error with offset %v read: %v n = %v err = %v\n", i, r, readableLength, err) } if input != nil { - if !bytes.Equal(output, input[i:]) { - tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output) + if !bytes.Equal(output[:readableLength], input[i:]) { + tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output[:readableLength]) } } } @@ -177,7 +177,8 @@ func TestDataAppend(t *testing.T) { input, found := tester.inputs[uint64(n)] var data io.Reader if !found { - data, input = GenerateRandomData(n) + input = testutil.RandomBytes(i, n) + data = bytes.NewReader(input) tester.inputs[uint64(n)] = input } else { data = io.LimitReader(bytes.NewReader(input), int64(n)) @@ -199,7 +200,8 @@ func TestDataAppend(t *testing.T) { appendInput, found := tester.inputs[uint64(m)] var appendData io.Reader if !found { - appendData, appendInput = GenerateRandomData(m) + appendInput = testutil.RandomBytes(i, m) + appendData = bytes.NewReader(appendInput) tester.inputs[uint64(m)] = appendInput } else { appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) @@ -232,7 +234,8 @@ func TestDataAppend(t *testing.T) { func TestRandomData(t *testing.T) { // This test can validate files up to a relatively short length, as tree chunker slows down drastically. // Validation of longer files is done by TestLocalStoreAndRetrieve in swarm package. - sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 524288, 524288 + 1, 524288 + 4097, 7 * 524288, 7*524288 + 1, 7*524288 + 4097} + //sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 524288, 524288 + 1, 524288 + 4097, 7 * 524288, 7*524288 + 1, 7*524288 + 4097} + sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4097, 8191, 8192, 12288, 12289, 524288} tester := &chunkerTester{t: t} for _, s := range sizes { @@ -271,7 +274,7 @@ func benchReadAll(reader LazySectionReader) { func benchmarkSplitJoin(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) ctx := context.TODO() @@ -291,7 +294,7 @@ func benchmarkSplitJoin(n int, t *testing.B) { func benchmarkSplitTreeSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() @@ -310,7 +313,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { func benchmarkSplitTreeBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() @@ -328,7 +331,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { func benchmarkSplitPyramidBMT(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) ctx := context.Background() @@ -346,7 +349,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { func benchmarkSplitPyramidSHA3(n int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) + data := testutil.RandomReader(i, n) putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) ctx := context.Background() @@ -364,8 +367,8 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { t.ReportAllocs() for i := 0; i < t.N; i++ { - data := testDataReader(n) - data1 := testDataReader(m) + data := testutil.RandomReader(i, n) + data1 := testutil.RandomReader(t.N+i, m) store := NewMapChunkStore() putGetter := newTestHasherStore(store, SHA3Hash) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 33133edd74..af104a5aeb 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -19,7 +19,6 @@ package storage import ( "bytes" "context" - "crypto/rand" "flag" "fmt" "io" @@ -31,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" ch "github.com/ethereum/go-ethereum/swarm/chunk" - colorable "github.com/mattn/go-colorable" + "github.com/mattn/go-colorable" ) var ( @@ -88,17 +87,6 @@ func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) return mput(store, n, GenerateRandomChunk) } -func mputChunks(store ChunkStore, chunks ...Chunk) error { - i := 0 - f := func(n int64) Chunk { - chunk := chunks[i] - i++ - return chunk - } - _, err := mput(store, len(chunks), f) - return err -} - func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error) { // put to localstore and wait for stored channel // does not check delivery error state @@ -162,10 +150,6 @@ func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error) return err } -func testDataReader(l int) (r io.Reader) { - return io.LimitReader(rand.Reader, int64(l)) -} - func (r *brokenLimitedReader) Read(buf []byte) (int, error) { if r.off+len(buf) > r.errAt { return 0, fmt.Errorf("Broken reader") diff --git a/swarm/storage/database.go b/swarm/storage/database.go index 3b5d003de1..e25fce31fb 100644 --- a/swarm/storage/database.go +++ b/swarm/storage/database.go @@ -20,8 +20,6 @@ package storage // no need for queueing/caching import ( - "fmt" - "github.com/ethereum/go-ethereum/metrics" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" @@ -46,13 +44,10 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { return database, nil } -func (db *LDBDatabase) Put(key []byte, value []byte) { +func (db *LDBDatabase) Put(key []byte, value []byte) error { metrics.GetOrRegisterCounter("ldbdatabase.put", nil).Inc(1) - err := db.db.Put(key, value, nil) - if err != nil { - fmt.Println("Error put", err) - } + return db.db.Put(key, value, nil) } func (db *LDBDatabase) Get(key []byte) ([]byte, error) { diff --git a/swarm/storage/encryption/encryption_test.go b/swarm/storage/encryption/encryption_test.go index c3abefdcec..0c0d0508c3 100644 --- a/swarm/storage/encryption/encryption_test.go +++ b/swarm/storage/encryption/encryption_test.go @@ -18,12 +18,12 @@ package encryption import ( "bytes" - "crypto/rand" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/testutil" ) var expectedTransformedHex = "352187af3a843decc63ceca6cb01ea39dbcf77caf0a8f705f5c30d557044ceec9392b94a79376f1e5c10cd0c0f2a98e5353bf22b3ea4fdac6677ee553dec192e3db64e179d0474e96088fb4abd2babd67de123fb398bdf84d818f7bda2c1ab60b3ea0e0569ae54aa969658eb4844e6960d2ff44d7c087ee3aaffa1c0ee5df7e50b615f7ad90190f022934ad5300c7d1809bfe71a11cc04cece5274eb97a5f20350630522c1dbb7cebaf4f97f84e03f5cfd88f2b48880b25d12f4d5e75c150f704ef6b46c72e07db2b705ac3644569dccd22fd8f964f6ef787fda63c46759af334e6f665f70eac775a7017acea49f3c7696151cb1b9434fa4ac27fb803921ffb5ec58dafa168098d7d5b97e384be3384cf5bc235c3d887fef89fe76c0065f9b8d6ad837b442340d9e797b46ef5709ea3358bc415df11e4830de986ef0f1c418ffdcc80e9a3cda9bea0ab5676c0d4240465c43ba527e3b4ea50b4f6255b510e5d25774a75449b0bd71e56c537ade4fcf0f4d63c99ae1dbb5a844971e2c19941b8facfcfc8ee3056e7cb3c7114c5357e845b52f7103cb6e00d2308c37b12baa5b769e1cc7b00fc06f2d16e70cc27a82cb9c1a4e40cb0d43907f73df2c9db44f1b51a6b0bc6d09f77ac3be14041fae3f9df2da42df43ae110904f9ecee278030185254d7c6e918a5512024d047f77a992088cb3190a6587aa54d0c7231c1cd2e455e0d4c07f74bece68e29cd8ba0190c0bcfb26d24634af5d91a81ef5d4dd3d614836ce942ddbf7bb1399317f4c03faa675f325f18324bf9433844bfe5c4cc04130c8d5c329562b7cd66e72f7355de8f5375a72202971613c32bd7f3fcdcd51080758cd1d0a46dbe8f0374381dbc359f5864250c63dde8131cbd7c98ae2b0147d6ea4bf65d1443d511b18e6d608bbb46ac036353b4c51df306a10a6f6939c38629a5c18aaf89cac04bd3ad5156e6b92011c88341cb08551bab0a89e6a46538f5af33b86121dba17e3a434c273f385cd2e8cb90bdd32747d8425d929ccbd9b0815c73325988855549a8489dfd047daf777aaa3099e54cf997175a5d9e1edfe363e3b68c70e02f6bf4fcde6a0f3f7d0e7e98bde1a72ae8b6cd27b32990680cc4a04fc467f41c5adcaddabfc71928a3f6872c360c1d765260690dd28b269864c8e380d9c92ef6b89b0094c8f9bb22608b4156381b19b920e9583c9616ce5693b4d2a6c689f02e6a91584a8e501e107403d2689dd0045269dd9946c0e969fb656a3b39d84a798831f5f9290f163eb2f97d3ae25071324e95e2256d9c1e56eb83c26397855323edc202d56ad05894333b7f0ed3c1e4734782eb8bd5477242fd80d7a89b12866f85cfae476322f032465d6b1253993033fccd4723530630ab97a1566460af9c90c9da843c229406e65f3fa578bd6bf04dee9b6153807ddadb8ceefc5c601a8ab26023c67b1ab1e8e0f29ce94c78c308005a781853e7a2e0e51738939a657c987b5e611f32f47b5ff461c52e63e0ea390515a8e1f5393dae54ea526934b5f310b76e3fa050e40718cb4c8a20e58946d6ee1879f08c52764422fe542b3240e75eccb7aa75b1f8a651e37a3bc56b0932cdae0e985948468db1f98eb4b77b82081ea25d8a762db00f7898864984bd80e2f3f35f236bf57291dec28f550769943bcfb6f884b7687589b673642ef7fe5d7d5a87d3eca5017f83ccb9a3310520474479464cb3f433440e7e2f1e28c0aef700a45848573409e7ab66e0cfd4fe5d2147ace81bc65fd8891f6245cd69246bbf5c27830e5ab882dd1d02aba34ff6ca9af88df00fd602892f02fedbdc65dedec203faf3f8ff4a97314e0ddb58b9ab756a61a562597f4088b445fcc3b28a708ca7b1485dcd791b779fbf2b3ef1ec5c6205f595fbe45a02105034147e5a146089c200a49dae33ae051a08ea5f974a21540aaeffa7f9d9e3d35478016fb27b871036eb27217a5b834b461f535752fb5f1c8dded3ae14ce3a2ef6639e2fe41939e3509e46e347a95d50b2080f1ba42c804b290ddc912c952d1cec3f2661369f738feacc0dbf1ea27429c644e45f9e26f30c341acd34c7519b2a1663e334621691e810767e9918c2c547b2e23cce915f97d26aac8d0d2fcd3edb7986ad4e2b8a852edebad534cb6c0e9f0797d3563e5409d7e068e48356c67ce519246cd9c560e881453df97cbba562018811e6cf8c327f399d1d1253ab47a19f4a0ccc7c6d86a9603e0551da310ea595d71305c4aad96819120a92cdbaf1f77ec8df9cc7c838c0d4de1e8692dd81da38268d1d71324bcffdafbe5122e4b81828e021e936d83ae8021eac592aa52cd296b5ce392c7173d622f8e07d18f59bb1b08ba15211af6703463b09b593af3c37735296816d9f2e7a369354a5374ea3955e14ca8ac56d5bfe4aef7a21bd825d6ae85530bee5d2aaaa4914981b3dfdb2e92ec2a27c83d74b59e84ff5c056f7d8945745f2efc3dcf28f288c6cd8383700fb2312f7001f24dd40015e436ae23e052fe9070ea9535b9c989898a9bda3d5382cf10e432fae6ccf0c825b3e6436edd3a9f8846e5606f8563931b5f29ba407c5236e5730225dda211a8504ec1817bc935e1fd9a532b648c502df302ed2063aed008fd5676131ac9e95998e9447b02bd29d77e38fcfd2959f2de929b31970335eb2a74348cc6918bc35b9bf749eab0fe304c946cd9e1ca284e6853c42646e60b6b39e0d3fb3c260abfc5c1b4ca3c3770f344118ca7c7f5c1ad1f123f8f369cd60afc3cdb3e9e81968c5c9fa7c8b014ffe0508dd4f0a2a976d5d1ca8fc9ad7a237d92cfe7b41413d934d6e142824b252699397e48e4bac4e91ebc10602720684bd0863773c548f9a2f9724245e47b129ecf65afd7252aac48c8a8d6fd3d888af592a01fb02dc71ed7538a700d3d16243e4621e0fcf9f8ed2b4e11c9fa9a95338bb1dac74a7d9bc4eb8cbf900b634a2a56469c00f5994e4f0934bdb947640e6d67e47d0b621aacd632bfd3c800bd7d93bd329f494a90e06ed51535831bd6e07ac1b4b11434ef3918fa9511813a002913f33f836454798b8d1787fea9a4c4743ba091ed192ed92f4d33e43a226bf9503e1a83a16dd340b3cbbf38af6db0d99201da8de529b4225f3d2fa2aad6621afc6c79ef3537720591edfc681ae6d00ede53ed724fc71b23b90d2e9b7158aaee98d626a4fe029107df2cb5f90147e07ebe423b1519d848af18af365c71bfd0665db46be493bbe99b79a188de0cf3594aef2299f0324075bdce9eb0b87bc29d62401ba4fd6ae48b1ba33261b5b845279becf38ee03e3dc5c45303321c5fac96fd02a3ad8c9e3b02127b320501333c9e6360440d1ad5e64a6239501502dde1a49c9abe33b66098458eee3d611bb06ffcd234a1b9aef4af5021cd61f0de6789f822ee116b5078aae8c129e8391d8987500d322b58edd1595dc570b57341f2df221b94a96ab7fbcf32a8ca9684196455694024623d7ed49f7d66e8dd453c0bae50e0d8b34377b22d0ece059e2c385dfc70b9089fcd27577c51f4d870b5738ee2b68c361a67809c105c7848b68860a829f29930857a9f9d40b14fd2384ac43bafdf43c0661103794c4bd07d1cfdd4681b6aeaefad53d4c1473359bcc5a83b09189352e5bb9a7498dd0effb89c35aad26954551f8b0621374b449bf515630bd3974dca982279733470fdd059aa9c3df403d8f22b38c4709c82d8f12b888e22990350490e16179caf406293cc9e65f116bafcbe96af132f679877061107a2f690a82a8cb46eea57a90abd23798c5937c6fe6b17be3f9bfa01ce117d2c268181b9095bf49f395fea07ca03838de0588c5e2db633e836d64488c1421e653ea52d810d096048c092d0da6e02fa6613890219f51a76148c8588c2487b171a28f17b7a299204874af0131725d793481333be5f08e86ca837a226850b0c1060891603bfecf9e55cddd22c0dbb28d495342d9cc3de8409f72e52a0115141cffe755c74f061c1a770428ccb0ae59536ee6fc074fbfc6cacb51a549d327527e20f8407477e60355863f1153f9ce95641198663c968874e7fdb29407bd771d94fdda8180cbb0358f5874738db705924b8cbe0cd5e1484aeb64542fe8f38667b7c34baf818c63b1e18440e9fba575254d063fd49f24ef26432f4eb323f3836972dca87473e3e9bb26dc3be236c3aae6bc8a6da567442309da0e8450e242fc9db836e2964f2c76a3b80a2c677979882dda7d7ebf62c93664018bcf4ec431fe6b403d49b3b36618b9c07c2d0d4569cb8d52223903debc72ec113955b206c34f1ae5300990ccfc0180f47d91afdb542b6312d12aeff7e19c645dc0b9fe6e3288e9539f6d5870f99882df187bfa6d24d179dfd1dac22212c8b5339f7171a3efc15b760fed8f68538bc5cbd845c2d1ab41f3a6c692820653eaef7930c02fbe6061d93805d73decdbb945572a7c44ed0241982a6e4d2d730898f82b3d9877cb7bca41cc6dcee67aa0c3d6db76f0b0a708ace0031113e48429de5d886c10e9200f68f32263a2fbf44a5992c2459fda7b8796ba796e3a0804fc25992ed2c9a5fe0580a6b809200ecde6caa0364b58be11564dcb9a616766dd7906db5636ee708b0204f38d309466d8d4a162965dd727e29f5a6c133e9b4ed5bafe803e479f9b2a7640c942c4a40b14ac7dc9828546052761a070f6404008f1ec3605836339c3da95a00b4fd81b2cabf88b51d2087d5b83e8c5b69bf96d8c72cbd278dad3bbb42b404b436f84ad688a22948adf60a81090f1e904291503c16e9f54b05fc76c881a5f95f0e732949e95d3f1bae2d3652a14fe0dda2d68879604657171856ef72637def2a96ac47d7b3fe86eb3198f5e0e626f06be86232305f2ae79ffcd2725e48208f9d8d63523f81915acc957563ab627cd6bc68c2a37d59fb0ed77a90aa9d085d6914a8ebada22a2c2d471b5163aeddd799d90fbb10ed6851ace2c4af504b7d572686700a59d6db46d5e42bb83f8e0c0ffe1dfa6582cc0b34c921ff6e85e83188d24906d5c08bb90069639e713051b3102b53e6f703e8210017878add5df68e6f2b108de279c5490e9eef5590185c4a1c744d4e00d244e1245a8805bd30407b1bc488db44870ccfd75a8af104df78efa2fb7ba31f048a263efdb3b63271fff4922bece9a71187108f65744a24f4947dc556b7440cb4fa45d296bb7f724588d1f245125b21ea063500029bd49650237f53899daf1312809552c81c5827341263cc807a29fe84746170cdfa1ff3838399a5645319bcaff674bb70efccdd88b3d3bb2f2d98111413585dc5d5bd5168f43b3f55e58972a5b2b9b3733febf02f931bd436648cb617c3794841aab961fe41277ab07812e1d3bc4ff6f4350a3e615bfba08c3b9480ef57904d3a16f7e916345202e3f93d11f7a7305170cb8c4eb9ac88ace8bbd1f377bdd5855d3162d6723d4435e84ce529b8f276a8927915ac759a0d04e5ca4a9d3da6291f0333b475df527e99fe38f7a4082662e8125936640c26dd1d17cf284ce6e2b17777a05aa0574f7793a6a062cc6f7263f7ab126b4528a17becfdec49ac0f7d8705aa1704af97fb861faa8a466161b2b5c08a5bacc79fe8500b913d65c8d3c52d1fd52d2ab2c9f52196e712455619c1cd3e0f391b274487944240e2ed8858dd0823c801094310024ae3fe4dd1cf5a2b6487b42cc5937bbafb193ee331d87e378258963d49b9da90899bbb4b88e79f78e866b0213f4719f67da7bcc2fce073c01e87c62ea3cdbcd589cfc41281f2f4c757c742d6d1e" @@ -125,8 +125,7 @@ func testEncryptDecryptIsIdentity(t *testing.T, padding int, initCtr uint32, dat key := GenerateRandomKey(keyLength) enc := New(key, padding, initCtr, hashFunc) - data := make([]byte, dataLength) - rand.Read(data) + data := testutil.RandomBytes(1, dataLength) encrypted, err := enc.Encrypt(data) if err != nil { diff --git a/swarm/storage/feed/binaryserializer.go b/swarm/storage/feed/binaryserializer.go new file mode 100644 index 0000000000..4e4f67a094 --- /dev/null +++ b/swarm/storage/feed/binaryserializer.go @@ -0,0 +1,44 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import "github.com/ethereum/go-ethereum/common/hexutil" + +type binarySerializer interface { + binaryPut(serializedData []byte) error + binaryLength() int + binaryGet(serializedData []byte) error +} + +// Values interface represents a string key-value store +// useful for building query strings +type Values interface { + Get(key string) string + Set(key, value string) +} + +type valueSerializer interface { + FromValues(values Values) error + AppendValues(values Values) +} + +// Hex serializes the structure and converts it to a hex string +func Hex(bin binarySerializer) string { + b := make([]byte, bin.binaryLength()) + bin.binaryPut(b) + return hexutil.Encode(b) +} diff --git a/swarm/storage/feed/binaryserializer_test.go b/swarm/storage/feed/binaryserializer_test.go new file mode 100644 index 0000000000..37828d1c94 --- /dev/null +++ b/swarm/storage/feed/binaryserializer_test.go @@ -0,0 +1,98 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +// KV mocks a key value store +type KV map[string]string + +func (kv KV) Get(key string) string { + return kv[key] +} +func (kv KV) Set(key, value string) { + kv[key] = value +} + +func compareByteSliceToExpectedHex(t *testing.T, variableName string, actualValue []byte, expectedHex string) { + if hexutil.Encode(actualValue) != expectedHex { + t.Fatalf("%s: Expected %s to be %s, got %s", t.Name(), variableName, expectedHex, hexutil.Encode(actualValue)) + } +} + +func testBinarySerializerRecovery(t *testing.T, bin binarySerializer, expectedHex string) { + name := reflect.TypeOf(bin).Elem().Name() + serialized := make([]byte, bin.binaryLength()) + if err := bin.binaryPut(serialized); err != nil { + t.Fatalf("%s.binaryPut error when trying to serialize structure: %s", name, err) + } + + compareByteSliceToExpectedHex(t, name, serialized, expectedHex) + + recovered := reflect.New(reflect.TypeOf(bin).Elem()).Interface().(binarySerializer) + if err := recovered.binaryGet(serialized); err != nil { + t.Fatalf("%s.binaryGet error when trying to deserialize structure: %s", name, err) + } + + if !reflect.DeepEqual(bin, recovered) { + t.Fatalf("Expected that the recovered %s equals the marshalled %s", name, name) + } + + serializedWrongLength := make([]byte, 1) + copy(serializedWrongLength[:], serialized) + if err := recovered.binaryGet(serializedWrongLength); err == nil { + t.Fatalf("Expected %s.binaryGet to fail since data is too small", name) + } +} + +func testBinarySerializerLengthCheck(t *testing.T, bin binarySerializer) { + name := reflect.TypeOf(bin).Elem().Name() + // make a slice that is too small to contain the metadata + serialized := make([]byte, bin.binaryLength()-1) + + if err := bin.binaryPut(serialized); err == nil { + t.Fatalf("Expected %s.binaryPut to fail, since target slice is too small", name) + } +} + +func testValueSerializer(t *testing.T, v valueSerializer, expected KV) { + name := reflect.TypeOf(v).Elem().Name() + kv := make(KV) + + v.AppendValues(kv) + if !reflect.DeepEqual(expected, kv) { + expj, _ := json.Marshal(expected) + gotj, _ := json.Marshal(kv) + t.Fatalf("Expected %s.AppendValues to return %s, got %s", name, string(expj), string(gotj)) + } + + recovered := reflect.New(reflect.TypeOf(v).Elem()).Interface().(valueSerializer) + err := recovered.FromValues(kv) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(recovered, v) { + t.Fatalf("Expected recovered %s to be the same", name) + } +} diff --git a/swarm/storage/feed/cacheentry.go b/swarm/storage/feed/cacheentry.go new file mode 100644 index 0000000000..be42008e99 --- /dev/null +++ b/swarm/storage/feed/cacheentry.go @@ -0,0 +1,48 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "bytes" + "context" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + hasherCount = 8 + feedsHashAlgorithm = storage.SHA3Hash + defaultRetrieveTimeout = 100 * time.Millisecond +) + +// cacheEntry caches the last known update of a specific Swarm feed. +type cacheEntry struct { + Update + *bytes.Reader + lastKey storage.Address +} + +// implements storage.LazySectionReader +func (r *cacheEntry) Size(ctx context.Context, _ chan bool) (int64, error) { + return int64(len(r.Update.data)), nil +} + +//returns the feed's topic +func (r *cacheEntry) Topic() Topic { + return r.Feed.Topic +} diff --git a/swarm/storage/feed/doc.go b/swarm/storage/feed/doc.go new file mode 100644 index 0000000000..1f07948f21 --- /dev/null +++ b/swarm/storage/feed/doc.go @@ -0,0 +1,43 @@ +/* +Package feeds defines Swarm Feeds. + +Swarm Feeds allows a user to build an update feed about a particular topic +without resorting to ENS on each update. +The update scheme is built on swarm chunks with chunk keys following +a predictable, versionable pattern. + +A Feed is tied to a unique identifier that is deterministically generated out of +the chosen topic. + +A Feed is defined as the series of updates of a specific user about a particular topic + +Actual data updates are also made in the form of swarm chunks. The keys +of the updates are the hash of a concatenation of properties as follows: + +updateAddr = H(Feed, Epoch ID) +where H is the SHA3 hash function +Feed is the combination of Topic and the user address +Epoch ID is a time slot. See the lookup package for more information. + +A user looking up a the latest update in a Feed only needs to know the Topic +and the other user's address. + +The Feed Update data is: +updatedata = Feed|Epoch|data + +The full update data that goes in the chunk payload is: +updatedata|sign(updatedata) + +Structure Summary: + +Request: Feed Update with signature + Update: headers + data + Header: Protocol version and reserved for future use placeholders + ID: Information about how to locate a specific update + Feed: Represents a user's series of publications about a specific Topic + Topic: Item that the updates are about + User: User who updates the Feed + Epoch: time slot where the update is stored + +*/ +package feed diff --git a/swarm/storage/mru/error.go b/swarm/storage/feed/error.go similarity index 87% rename from swarm/storage/mru/error.go rename to swarm/storage/feed/error.go index 18ab525582..206ba3316f 100644 --- a/swarm/storage/mru/error.go +++ b/swarm/storage/feed/error.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package mru +package feed import ( "fmt" @@ -35,7 +35,7 @@ const ( ErrCnt ) -// Error is a the typed error object used for Mutable Resources +// Error is a the typed error object used for Swarm feeds type Error struct { code int err string @@ -47,12 +47,12 @@ func (e *Error) Error() string { } // Code returns the error code -// Error codes are enumerated in the error.go file within the mru package +// Error codes are enumerated in the error.go file within the feeds package func (e *Error) Code() int { return e.code } -// NewError creates a new Mutable Resource Error object with the specified code and custom error message +// NewError creates a new Swarm feeds Error object with the specified code and custom error message func NewError(code int, s string) error { if code < 0 || code >= ErrCnt { panic("no such error code!") diff --git a/swarm/storage/feed/feed.go b/swarm/storage/feed/feed.go new file mode 100644 index 0000000000..b6ea665a6f --- /dev/null +++ b/swarm/storage/feed/feed.go @@ -0,0 +1,125 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "hash" + "unsafe" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// Feed represents a particular user's stream of updates on a topic +type Feed struct { + Topic Topic `json:"topic"` + User common.Address `json:"user"` +} + +// Feed layout: +// TopicLength bytes +// userAddr common.AddressLength bytes +const feedLength = TopicLength + common.AddressLength + +// mapKey calculates a unique id for this feed. Used by the cache map in `Handler` +func (f *Feed) mapKey() uint64 { + serializedData := make([]byte, feedLength) + f.binaryPut(serializedData) + hasher := hashPool.Get().(hash.Hash) + defer hashPool.Put(hasher) + hasher.Reset() + hasher.Write(serializedData) + hash := hasher.Sum(nil) + return *(*uint64)(unsafe.Pointer(&hash[0])) +} + +// binaryPut serializes this feed instance into the provided slice +func (f *Feed) binaryPut(serializedData []byte) error { + if len(serializedData) != feedLength { + return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize feed. Expected %d, got %d", feedLength, len(serializedData)) + } + var cursor int + copy(serializedData[cursor:cursor+TopicLength], f.Topic[:TopicLength]) + cursor += TopicLength + + copy(serializedData[cursor:cursor+common.AddressLength], f.User[:]) + cursor += common.AddressLength + + return nil +} + +// binaryLength returns the expected size of this structure when serialized +func (f *Feed) binaryLength() int { + return feedLength +} + +// binaryGet restores the current instance from the information contained in the passed slice +func (f *Feed) binaryGet(serializedData []byte) error { + if len(serializedData) != feedLength { + return NewErrorf(ErrInvalidValue, "Incorrect slice size to read feed. Expected %d, got %d", feedLength, len(serializedData)) + } + + var cursor int + copy(f.Topic[:], serializedData[cursor:cursor+TopicLength]) + cursor += TopicLength + + copy(f.User[:], serializedData[cursor:cursor+common.AddressLength]) + cursor += common.AddressLength + + return nil +} + +// Hex serializes the feed to a hex string +func (f *Feed) Hex() string { + serializedData := make([]byte, feedLength) + f.binaryPut(serializedData) + return hexutil.Encode(serializedData) +} + +// FromValues deserializes this instance from a string key-value store +// useful to parse query strings +func (f *Feed) FromValues(values Values) (err error) { + topic := values.Get("topic") + if topic != "" { + if err := f.Topic.FromHex(values.Get("topic")); err != nil { + return err + } + } else { // see if the user set name and relatedcontent + name := values.Get("name") + relatedContent, _ := hexutil.Decode(values.Get("relatedcontent")) + if len(relatedContent) > 0 { + if len(relatedContent) < storage.AddressLength { + return NewErrorf(ErrInvalidValue, "relatedcontent field must be a hex-encoded byte array exactly %d bytes long", storage.AddressLength) + } + relatedContent = relatedContent[:storage.AddressLength] + } + f.Topic, err = NewTopic(name, relatedContent) + if err != nil { + return err + } + } + f.User = common.HexToAddress(values.Get("user")) + return nil +} + +// AppendValues serializes this structure into the provided string key-value store +// useful to build query strings +func (f *Feed) AppendValues(values Values) { + values.Set("topic", f.Topic.Hex()) + values.Set("user", f.User.Hex()) +} diff --git a/swarm/storage/feed/feed_test.go b/swarm/storage/feed/feed_test.go new file mode 100644 index 0000000000..6a575594f4 --- /dev/null +++ b/swarm/storage/feed/feed_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package feed + +import ( + "testing" +) + +func getTestFeed() *Feed { + topic, _ := NewTopic("world news report, every hour", nil) + return &Feed{ + Topic: topic, + User: newCharlieSigner().Address(), + } +} + +func TestFeedSerializerDeserializer(t *testing.T) { + testBinarySerializerRecovery(t, getTestFeed(), "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781c") +} + +func TestFeedSerializerLengthCheck(t *testing.T) { + testBinarySerializerLengthCheck(t, getTestFeed()) +} diff --git a/swarm/storage/feed/handler.go b/swarm/storage/feed/handler.go new file mode 100644 index 0000000000..9e2640282c --- /dev/null +++ b/swarm/storage/feed/handler.go @@ -0,0 +1,295 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Handler is the API for feeds +// It enables creating, updating, syncing and retrieving feed updates and their data +package feed + +import ( + "bytes" + "context" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +type Handler struct { + chunkStore *storage.NetStore + HashSize int + cache map[uint64]*cacheEntry + cacheLock sync.RWMutex + storeTimeout time.Duration + queryMaxPeriods uint32 +} + +// HandlerParams pass parameters to the Handler constructor NewHandler +// Signer and TimestampProvider are mandatory parameters +type HandlerParams struct { +} + +// hashPool contains a pool of ready hashers +var hashPool sync.Pool + +// init initializes the package and hashPool +func init() { + hashPool = sync.Pool{ + New: func() interface{} { + return storage.MakeHashFunc(feedsHashAlgorithm)() + }, + } +} + +// NewHandler creates a new Swarm feeds API +func NewHandler(params *HandlerParams) *Handler { + fh := &Handler{ + cache: make(map[uint64]*cacheEntry), + } + + for i := 0; i < hasherCount; i++ { + hashfunc := storage.MakeHashFunc(feedsHashAlgorithm)() + if fh.HashSize == 0 { + fh.HashSize = hashfunc.Size() + } + hashPool.Put(hashfunc) + } + + return fh +} + +// SetStore sets the store backend for the Swarm feeds API +func (h *Handler) SetStore(store *storage.NetStore) { + h.chunkStore = store +} + +// Validate is a chunk validation method +// If it looks like a feed update, the chunk address is checked against the userAddr of the update's signature +// It implements the storage.ChunkValidator interface +func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { + dataLength := len(data) + if dataLength < minimumSignedUpdateLength { + return false + } + + // check if it is a properly formatted update chunk with + // valid signature and proof of ownership of the feed it is trying + // to update + + // First, deserialize the chunk + var r Request + if err := r.fromChunk(chunkAddr, data); err != nil { + log.Debug("Invalid feed update chunk", "addr", chunkAddr.Hex(), "err", err.Error()) + return false + } + + // Verify signatures and that the signer actually owns the feed + // If it fails, it means either the signature is not valid, data is corrupted + // or someone is trying to update someone else's feed. + if err := r.Verify(); err != nil { + log.Debug("Invalid feed update signature", "err", err) + return false + } + + return true +} + +// GetContent retrieves the data payload of the last synced update of the feed +func (h *Handler) GetContent(feed *Feed) (storage.Address, []byte, error) { + if feed == nil { + return nil, nil, NewError(ErrInvalidValue, "feed is nil") + } + feedUpdate := h.get(feed) + if feedUpdate == nil { + return nil, nil, NewError(ErrNotFound, "feed update not cached") + } + return feedUpdate.lastKey, feedUpdate.data, nil +} + +// NewRequest prepares a Request structure with all the necessary information to +// just add the desired data and sign it. +// The resulting structure can then be signed and passed to Handler.Update to be verified and sent +func (h *Handler) NewRequest(ctx context.Context, feed *Feed) (request *Request, err error) { + if feed == nil { + return nil, NewError(ErrInvalidValue, "feed cannot be nil") + } + + now := TimestampProvider.Now().Time + request = new(Request) + request.Header.Version = ProtocolVersion + + query := NewQueryLatest(feed, lookup.NoClue) + + feedUpdate, err := h.Lookup(ctx, query) + if err != nil { + if err.(*Error).code != ErrNotFound { + return nil, err + } + // not finding updates means that there is a network error + // or that the feed really does not have updates + } + + request.Feed = *feed + + // if we already have an update, then find next epoch + if feedUpdate != nil { + request.Epoch = lookup.GetNextEpoch(feedUpdate.Epoch, now) + } else { + request.Epoch = lookup.GetFirstEpoch(now) + } + + return request, nil +} + +// Lookup retrieves a specific or latest feed update +// Lookup works differently depending on the configuration of `query` +// See the `query` documentation and helper functions: +// `NewQueryLatest` and `NewQuery` +func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) { + + timeLimit := query.TimeLimit + if timeLimit == 0 { // if time limit is set to zero, the user wants to get the latest update + timeLimit = TimestampProvider.Now().Time + } + + if query.Hint == lookup.NoClue { // try to use our cache + entry := h.get(&query.Feed) + if entry != nil && entry.Epoch.Time <= timeLimit { // avoid bad hints + query.Hint = entry.Epoch + } + } + + // we can't look for anything without a store + if h.chunkStore == nil { + return nil, NewError(ErrInit, "Call Handler.SetStore() before performing lookups") + } + + var id ID + id.Feed = query.Feed + var readCount int + + // Invoke the lookup engine. + // The callback will be called every time the lookup algorithm needs to guess + requestPtr, err := lookup.Lookup(timeLimit, query.Hint, func(epoch lookup.Epoch, now uint64) (interface{}, error) { + readCount++ + id.Epoch = epoch + ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout) + defer cancel() + + chunk, err := h.chunkStore.Get(ctx, id.Addr()) + if err != nil { // TODO: check for catastrophic errors other than chunk not found + return nil, nil + } + + var request Request + if err := request.fromChunk(chunk.Address(), chunk.Data()); err != nil { + return nil, nil + } + if request.Time <= timeLimit { + return &request, nil + } + return nil, nil + }) + if err != nil { + return nil, err + } + + log.Info(fmt.Sprintf("Feed lookup finished in %d lookups", readCount)) + + request, _ := requestPtr.(*Request) + if request == nil { + return nil, NewError(ErrNotFound, "no feed updates found") + } + return h.updateCache(request) + +} + +// update feed updates cache with specified content +func (h *Handler) updateCache(request *Request) (*cacheEntry, error) { + + updateAddr := request.Addr() + log.Trace("feed cache update", "topic", request.Topic.Hex(), "updateaddr", updateAddr, "epoch time", request.Epoch.Time, "epoch level", request.Epoch.Level) + + feedUpdate := h.get(&request.Feed) + if feedUpdate == nil { + feedUpdate = &cacheEntry{} + h.set(&request.Feed, feedUpdate) + } + + // update our rsrcs entry map + feedUpdate.lastKey = updateAddr + feedUpdate.Update = request.Update + feedUpdate.Reader = bytes.NewReader(feedUpdate.data) + return feedUpdate, nil +} + +// Update publishes a feed update +// Note that a feed update cannot span chunks, and thus has a MAX NET LENGTH 4096, INCLUDING update header data and signature. +// This results in a max payload of `maxUpdateDataLength` (check update.go for more details) +// An error will be returned if the total length of the chunk payload will exceed this limit. +// Update can only check if the caller is trying to overwrite the very last known version, otherwise it just puts the update +// on the network. +func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Address, err error) { + + // we can't update anything without a store + if h.chunkStore == nil { + return nil, NewError(ErrInit, "Call Handler.SetStore() before updating") + } + + feedUpdate := h.get(&r.Feed) + if feedUpdate != nil && feedUpdate.Epoch.Equals(r.Epoch) { // This is the only cheap check we can do for sure + return nil, NewError(ErrInvalidValue, "A former update in this epoch is already known to exist") + } + + chunk, err := r.toChunk() // Serialize the update into a chunk. Fails if data is too big + if err != nil { + return nil, err + } + + // send the chunk + h.chunkStore.Put(ctx, chunk) + log.Trace("feed update", "updateAddr", r.idAddr, "epoch time", r.Epoch.Time, "epoch level", r.Epoch.Level, "data", chunk.Data()) + // update our feed updates map cache entry if the new update is older than the one we have, if we have it. + if feedUpdate != nil && r.Epoch.After(feedUpdate.Epoch) { + feedUpdate.Epoch = r.Epoch + feedUpdate.data = make([]byte, len(r.data)) + feedUpdate.lastKey = r.idAddr + copy(feedUpdate.data, r.data) + feedUpdate.Reader = bytes.NewReader(feedUpdate.data) + } + + return r.idAddr, nil +} + +// Retrieves the feed update cache value for the given nameHash +func (h *Handler) get(feed *Feed) *cacheEntry { + mapKey := feed.mapKey() + h.cacheLock.RLock() + defer h.cacheLock.RUnlock() + feedUpdate := h.cache[mapKey] + return feedUpdate +} + +// Sets the feed update cache value for the given feed +func (h *Handler) set(feed *Feed, feedUpdate *cacheEntry) { + mapKey := feed.mapKey() + h.cacheLock.Lock() + defer h.cacheLock.Unlock() + h.cache[mapKey] = feedUpdate +} diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go new file mode 100644 index 0000000000..fb2ef3a6ba --- /dev/null +++ b/swarm/storage/feed/handler_test.go @@ -0,0 +1,507 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "bytes" + "context" + "flag" + "fmt" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +var ( + loglevel = flag.Int("loglevel", 3, "loglevel") + startTime = Timestamp{ + Time: uint64(4200), + } + cleanF func() + subtopicName = "føø.bar" + hashfunc = storage.MakeHashFunc(storage.DefaultHash) +) + +func init() { + flag.Parse() + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + +// simulated timeProvider +type fakeTimeProvider struct { + currentTime uint64 +} + +func (f *fakeTimeProvider) Tick() { + f.currentTime++ +} + +func (f *fakeTimeProvider) Set(time uint64) { + f.currentTime = time +} + +func (f *fakeTimeProvider) FastForward(offset uint64) { + f.currentTime += offset +} + +func (f *fakeTimeProvider) Now() Timestamp { + return Timestamp{ + Time: f.currentTime, + } +} + +// make updates and retrieve them based on periods and versions +func TestFeedsHandler(t *testing.T) { + + // make fake timeProvider + clock := &fakeTimeProvider{ + currentTime: startTime.Time, // clock starts at t=4200 + } + + // signer containing private key + signer := newAliceSigner() + + feedsHandler, datadir, teardownTest, err := setupTest(clock, signer) + if err != nil { + t.Fatal(err) + } + defer teardownTest() + + // create a new feed + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + topic, _ := NewTopic("Mess with Swarm feeds code and see what ghost catches you", nil) + fd := Feed{ + Topic: topic, + User: signer.Address(), + } + + // data for updates: + updates := []string{ + "blinky", // t=4200 + "pinky", // t=4242 + "inky", // t=4284 + "clyde", // t=4285 + } + + request := NewFirstRequest(fd.Topic) // this timestamps the update at t = 4200 (start time) + chunkAddress := make(map[string]storage.Address) + data := []byte(updates[0]) + request.SetData(data) + if err := request.Sign(signer); err != nil { + t.Fatal(err) + } + chunkAddress[updates[0]], err = feedsHandler.Update(ctx, request) + if err != nil { + t.Fatal(err) + } + + // move the clock ahead 21 seconds + clock.FastForward(21) // t=4221 + + request, err = feedsHandler.NewRequest(ctx, &request.Feed) // this timestamps the update at t = 4221 + if err != nil { + t.Fatal(err) + } + if request.Epoch.Base() != 0 || request.Epoch.Level != lookup.HighestLevel-1 { + t.Fatalf("Suggested epoch BaseTime should be 0 and Epoch level should be %d", lookup.HighestLevel-1) + } + + request.Epoch.Level = lookup.HighestLevel // force level 25 instead of 24 to make it fail + data = []byte(updates[1]) + request.SetData(data) + if err := request.Sign(signer); err != nil { + t.Fatal(err) + } + chunkAddress[updates[1]], err = feedsHandler.Update(ctx, request) + if err == nil { + t.Fatal("Expected update to fail since an update in this epoch already exists") + } + + // move the clock ahead 21 seconds + clock.FastForward(21) // t=4242 + request, err = feedsHandler.NewRequest(ctx, &request.Feed) + if err != nil { + t.Fatal(err) + } + request.SetData(data) + if err := request.Sign(signer); err != nil { + t.Fatal(err) + } + chunkAddress[updates[1]], err = feedsHandler.Update(ctx, request) + if err != nil { + t.Fatal(err) + } + + // move the clock ahead 42 seconds + clock.FastForward(42) // t=4284 + request, err = feedsHandler.NewRequest(ctx, &request.Feed) + if err != nil { + t.Fatal(err) + } + data = []byte(updates[2]) + request.SetData(data) + if err := request.Sign(signer); err != nil { + t.Fatal(err) + } + chunkAddress[updates[2]], err = feedsHandler.Update(ctx, request) + if err != nil { + t.Fatal(err) + } + + // move the clock ahead 1 second + clock.FastForward(1) // t=4285 + request, err = feedsHandler.NewRequest(ctx, &request.Feed) + if err != nil { + t.Fatal(err) + } + if request.Epoch.Base() != 0 || request.Epoch.Level != 22 { + t.Fatalf("Expected epoch base time to be %d, got %d. Expected epoch level to be %d, got %d", 0, request.Epoch.Base(), 22, request.Epoch.Level) + } + data = []byte(updates[3]) + request.SetData(data) + + if err := request.Sign(signer); err != nil { + t.Fatal(err) + } + chunkAddress[updates[3]], err = feedsHandler.Update(ctx, request) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Second) + feedsHandler.Close() + + // check we can retrieve the updates after close + clock.FastForward(2000) // t=6285 + + feedParams := &HandlerParams{} + + feedsHandler2, err := NewTestHandler(datadir, feedParams) + if err != nil { + t.Fatal(err) + } + + update2, err := feedsHandler2.Lookup(ctx, NewQueryLatest(&request.Feed, lookup.NoClue)) + if err != nil { + t.Fatal(err) + } + + // last update should be "clyde" + if !bytes.Equal(update2.data, []byte(updates[len(updates)-1])) { + t.Fatalf("feed update data was %v, expected %v", string(update2.data), updates[len(updates)-1]) + } + if update2.Level != 22 { + t.Fatalf("feed update epoch level was %d, expected 22", update2.Level) + } + if update2.Base() != 0 { + t.Fatalf("feed update epoch base time was %d, expected 0", update2.Base()) + } + log.Debug("Latest lookup", "epoch base time", update2.Base(), "epoch level", update2.Level, "data", update2.data) + + // specific point in time + update, err := feedsHandler2.Lookup(ctx, NewQuery(&request.Feed, 4284, lookup.NoClue)) + if err != nil { + t.Fatal(err) + } + // check data + if !bytes.Equal(update.data, []byte(updates[2])) { + t.Fatalf("feed update data (historical) was %v, expected %v", string(update2.data), updates[2]) + } + log.Debug("Historical lookup", "epoch base time", update2.Base(), "epoch level", update2.Level, "data", update2.data) + + // beyond the first should yield an error + update, err = feedsHandler2.Lookup(ctx, NewQuery(&request.Feed, startTime.Time-1, lookup.NoClue)) + if err == nil { + t.Fatalf("expected previous to fail, returned epoch %s data %v", update.Epoch.String(), update.data) + } + +} + +const Day = 60 * 60 * 24 +const Year = Day * 365 +const Month = Day * 30 + +func generateData(x uint64) []byte { + return []byte(fmt.Sprintf("%d", x)) +} + +func TestSparseUpdates(t *testing.T) { + + // make fake timeProvider + timeProvider := &fakeTimeProvider{ + currentTime: startTime.Time, + } + + // signer containing private key + signer := newAliceSigner() + + rh, datadir, teardownTest, err := setupTest(timeProvider, signer) + if err != nil { + t.Fatal(err) + } + defer teardownTest() + defer os.RemoveAll(datadir) + + // create a new feed + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + topic, _ := NewTopic("Very slow updates", nil) + fd := Feed{ + Topic: topic, + User: signer.Address(), + } + + // publish one update every 5 years since Unix 0 until today + today := uint64(1533799046) + var epoch lookup.Epoch + var lastUpdateTime uint64 + for T := uint64(0); T < today; T += 5 * Year { + request := NewFirstRequest(fd.Topic) + request.Epoch = lookup.GetNextEpoch(epoch, T) + request.data = generateData(T) // this generates some data that depends on T, so we can check later + request.Sign(signer) + if err != nil { + t.Fatal(err) + } + + if _, err := rh.Update(ctx, request); err != nil { + t.Fatal(err) + } + epoch = request.Epoch + lastUpdateTime = T + } + + query := NewQuery(&fd, today, lookup.NoClue) + + _, err = rh.Lookup(ctx, query) + if err != nil { + t.Fatal(err) + } + + _, content, err := rh.GetContent(&fd) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(generateData(lastUpdateTime), content) { + t.Fatalf("Expected to recover last written value %d, got %s", lastUpdateTime, string(content)) + } + + // lookup the closest update to 35*Year + 6* Month (~ June 2005): + // it should find the update we put on 35*Year, since we were updating every 5 years. + + query.TimeLimit = 35*Year + 6*Month + + _, err = rh.Lookup(ctx, query) + if err != nil { + t.Fatal(err) + } + + _, content, err = rh.GetContent(&fd) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(generateData(35*Year), content) { + t.Fatalf("Expected to recover %d, got %s", 35*Year, string(content)) + } +} + +func TestValidator(t *testing.T) { + + // make fake timeProvider + timeProvider := &fakeTimeProvider{ + currentTime: startTime.Time, + } + + // signer containing private key. Alice will be the good girl + signer := newAliceSigner() + + // set up sim timeProvider + rh, _, teardownTest, err := setupTest(timeProvider, signer) + if err != nil { + t.Fatal(err) + } + defer teardownTest() + + // create new feed + topic, _ := NewTopic(subtopicName, nil) + fd := Feed{ + Topic: topic, + User: signer.Address(), + } + mr := NewFirstRequest(fd.Topic) + + // chunk with address + data := []byte("foo") + mr.SetData(data) + if err := mr.Sign(signer); err != nil { + t.Fatalf("sign fail: %v", err) + } + + chunk, err := mr.toChunk() + if err != nil { + t.Fatal(err) + } + if !rh.Validate(chunk.Address(), chunk.Data()) { + t.Fatal("Chunk validator fail on update chunk") + } + + address := chunk.Address() + // mess with the address + address[0] = 11 + address[15] = 99 + + if rh.Validate(address, chunk.Data()) { + t.Fatal("Expected Validate to fail with false chunk address") + } +} + +// tests that the content address validator correctly checks the data +// tests that feed update chunks are passed through content address validator +// there is some redundancy in this test as it also tests content addressed chunks, +// which should be evaluated as invalid chunks by this validator +func TestValidatorInStore(t *testing.T) { + + // make fake timeProvider + TimestampProvider = &fakeTimeProvider{ + currentTime: startTime.Time, + } + + // signer containing private key + signer := newAliceSigner() + + // set up localstore + datadir, err := ioutil.TempDir("", "storage-testfeedsvalidator") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(datadir) + + handlerParams := storage.NewDefaultLocalStoreParams() + handlerParams.Init(datadir) + store, err := storage.NewLocalStore(handlerParams, nil) + if err != nil { + t.Fatal(err) + } + + // set up Swarm feeds handler and add is as a validator to the localstore + fhParams := &HandlerParams{} + fh := NewHandler(fhParams) + store.Validators = append(store.Validators, fh) + + // create content addressed chunks, one good, one faulty + chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) + goodChunk := chunks[0] + badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data()) + + topic, _ := NewTopic("xyzzy", nil) + fd := Feed{ + Topic: topic, + User: signer.Address(), + } + + // create a feed update chunk with correct publickey + id := ID{ + Epoch: lookup.Epoch{Time: 42, + Level: 1, + }, + Feed: fd, + } + + updateAddr := id.Addr() + data := []byte("bar") + + r := new(Request) + r.idAddr = updateAddr + r.Update.ID = id + r.data = data + + r.Sign(signer) + + uglyChunk, err := r.toChunk() + if err != nil { + t.Fatal(err) + } + + // put the chunks in the store and check their error status + err = store.Put(context.Background(), goodChunk) + if err == nil { + t.Fatal("expected error on good content address chunk with feed update validator only, but got nil") + } + err = store.Put(context.Background(), badChunk) + if err == nil { + t.Fatal("expected error on bad content address chunk with feed update validator only, but got nil") + } + err = store.Put(context.Background(), uglyChunk) + if err != nil { + t.Fatalf("expected no error on feed update chunk with feed update validator only, but got: %s", err) + } +} + +// create rpc and feeds Handler +func setupTest(timeProvider timestampProvider, signer Signer) (fh *TestHandler, datadir string, teardown func(), err error) { + + var fsClean func() + var rpcClean func() + cleanF = func() { + if fsClean != nil { + fsClean() + } + if rpcClean != nil { + rpcClean() + } + } + + // temp datadir + datadir, err = ioutil.TempDir("", "fh") + if err != nil { + return nil, "", nil, err + } + fsClean = func() { + os.RemoveAll(datadir) + } + + TimestampProvider = timeProvider + fhParams := &HandlerParams{} + fh, err = NewTestHandler(datadir, fhParams) + return fh, datadir, cleanF, err +} + +func newAliceSigner() *GenericSigner { + privKey, _ := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + return NewGenericSigner(privKey) +} + +func newBobSigner() *GenericSigner { + privKey, _ := crypto.HexToECDSA("accedeaccedeaccedeaccedeaccedeaccedeaccedeaccedeaccedeaccedecaca") + return NewGenericSigner(privKey) +} + +func newCharlieSigner() *GenericSigner { + privKey, _ := crypto.HexToECDSA("facadefacadefacadefacadefacadefacadefacadefacadefacadefacadefaca") + return NewGenericSigner(privKey) +} diff --git a/swarm/storage/feed/id.go b/swarm/storage/feed/id.go new file mode 100644 index 0000000000..7e17743c11 --- /dev/null +++ b/swarm/storage/feed/id.go @@ -0,0 +1,123 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "fmt" + "hash" + "strconv" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// ID uniquely identifies an update on the network. +type ID struct { + Feed `json:"feed"` + lookup.Epoch `json:"epoch"` +} + +// ID layout: +// Feed feedLength bytes +// Epoch EpochLength +const idLength = feedLength + lookup.EpochLength + +// Addr calculates the feed update chunk address corresponding to this ID +func (u *ID) Addr() (updateAddr storage.Address) { + serializedData := make([]byte, idLength) + var cursor int + u.Feed.binaryPut(serializedData[cursor : cursor+feedLength]) + cursor += feedLength + + eid := u.Epoch.ID() + copy(serializedData[cursor:cursor+lookup.EpochLength], eid[:]) + + hasher := hashPool.Get().(hash.Hash) + defer hashPool.Put(hasher) + hasher.Reset() + hasher.Write(serializedData) + return hasher.Sum(nil) +} + +// binaryPut serializes this instance into the provided slice +func (u *ID) binaryPut(serializedData []byte) error { + if len(serializedData) != idLength { + return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize ID. Expected %d, got %d", idLength, len(serializedData)) + } + var cursor int + if err := u.Feed.binaryPut(serializedData[cursor : cursor+feedLength]); err != nil { + return err + } + cursor += feedLength + + epochBytes, err := u.Epoch.MarshalBinary() + if err != nil { + return err + } + copy(serializedData[cursor:cursor+lookup.EpochLength], epochBytes[:]) + cursor += lookup.EpochLength + + return nil +} + +// binaryLength returns the expected size of this structure when serialized +func (u *ID) binaryLength() int { + return idLength +} + +// binaryGet restores the current instance from the information contained in the passed slice +func (u *ID) binaryGet(serializedData []byte) error { + if len(serializedData) != idLength { + return NewErrorf(ErrInvalidValue, "Incorrect slice size to read ID. Expected %d, got %d", idLength, len(serializedData)) + } + + var cursor int + if err := u.Feed.binaryGet(serializedData[cursor : cursor+feedLength]); err != nil { + return err + } + cursor += feedLength + + if err := u.Epoch.UnmarshalBinary(serializedData[cursor : cursor+lookup.EpochLength]); err != nil { + return err + } + cursor += lookup.EpochLength + + return nil +} + +// FromValues deserializes this instance from a string key-value store +// useful to parse query strings +func (u *ID) FromValues(values Values) error { + level, _ := strconv.ParseUint(values.Get("level"), 10, 32) + u.Epoch.Level = uint8(level) + u.Epoch.Time, _ = strconv.ParseUint(values.Get("time"), 10, 64) + + if u.Feed.User == (common.Address{}) { + return u.Feed.FromValues(values) + } + return nil +} + +// AppendValues serializes this structure into the provided string key-value store +// useful to build query strings +func (u *ID) AppendValues(values Values) { + values.Set("level", fmt.Sprintf("%d", u.Epoch.Level)) + values.Set("time", fmt.Sprintf("%d", u.Epoch.Time)) + u.Feed.AppendValues(values) +} diff --git a/swarm/storage/feed/id_test.go b/swarm/storage/feed/id_test.go new file mode 100644 index 0000000000..e561ff9b46 --- /dev/null +++ b/swarm/storage/feed/id_test.go @@ -0,0 +1,28 @@ +package feed + +import ( + "testing" + + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +func getTestID() *ID { + return &ID{ + Feed: *getTestFeed(), + Epoch: lookup.GetFirstEpoch(1000), + } +} + +func TestIDAddr(t *testing.T) { + id := getTestID() + updateAddr := id.Addr() + compareByteSliceToExpectedHex(t, "updateAddr", updateAddr, "0x8b24583ec293e085f4c78aaee66d1bc5abfb8b4233304d14a349afa57af2a783") +} + +func TestIDSerializer(t *testing.T) { + testBinarySerializerRecovery(t, getTestID(), "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019") +} + +func TestIDLengthCheck(t *testing.T) { + testBinarySerializerLengthCheck(t, getTestID()) +} diff --git a/swarm/storage/feed/lookup/epoch.go b/swarm/storage/feed/lookup/epoch.go new file mode 100644 index 0000000000..bafe954779 --- /dev/null +++ b/swarm/storage/feed/lookup/epoch.go @@ -0,0 +1,91 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lookup + +import ( + "encoding/binary" + "errors" + "fmt" +) + +// Epoch represents a time slot at a particular frequency level +type Epoch struct { + Time uint64 `json:"time"` // Time stores the time at which the update or lookup takes place + Level uint8 `json:"level"` // Level indicates the frequency level as the exponent of a power of 2 +} + +// EpochID is a unique identifier for an Epoch, based on its level and base time. +type EpochID [8]byte + +// EpochLength stores the serialized binary length of an Epoch +const EpochLength = 8 + +// MaxTime contains the highest possible time value an Epoch can handle +const MaxTime uint64 = (1 << 56) - 1 + +// Base returns the base time of the Epoch +func (e *Epoch) Base() uint64 { + return getBaseTime(e.Time, e.Level) +} + +// ID Returns the unique identifier of this epoch +func (e *Epoch) ID() EpochID { + base := e.Base() + var id EpochID + binary.LittleEndian.PutUint64(id[:], base) + id[7] = e.Level + return id +} + +// MarshalBinary implements the encoding.BinaryMarshaller interface +func (e *Epoch) MarshalBinary() (data []byte, err error) { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b[:], e.Time) + b[7] = e.Level + return b, nil +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaller interface +func (e *Epoch) UnmarshalBinary(data []byte) error { + if len(data) != EpochLength { + return errors.New("Invalid data unmarshalling Epoch") + } + b := make([]byte, 8) + copy(b, data) + e.Level = b[7] + b[7] = 0 + e.Time = binary.LittleEndian.Uint64(b) + return nil +} + +// After returns true if this epoch occurs later or exactly at the other epoch. +func (e *Epoch) After(epoch Epoch) bool { + if e.Time == epoch.Time { + return e.Level < epoch.Level + } + return e.Time >= epoch.Time +} + +// Equals compares two epochs and returns true if they refer to the same time period. +func (e *Epoch) Equals(epoch Epoch) bool { + return e.Level == epoch.Level && e.Base() == epoch.Base() +} + +// String implements the Stringer interface. +func (e *Epoch) String() string { + return fmt.Sprintf("Epoch{Time:%d, Level:%d}", e.Time, e.Level) +} diff --git a/swarm/storage/feed/lookup/epoch_test.go b/swarm/storage/feed/lookup/epoch_test.go new file mode 100644 index 0000000000..0629f3d1df --- /dev/null +++ b/swarm/storage/feed/lookup/epoch_test.go @@ -0,0 +1,57 @@ +package lookup_test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +func TestMarshallers(t *testing.T) { + + for i := uint64(1); i < lookup.MaxTime; i *= 3 { + e := lookup.Epoch{ + Time: i, + Level: uint8(i % 20), + } + b, err := e.MarshalBinary() + if err != nil { + t.Fatal(err) + } + var e2 lookup.Epoch + if err := e2.UnmarshalBinary(b); err != nil { + t.Fatal(err) + } + if e != e2 { + t.Fatal("Expected unmarshalled epoch to be equal to marshalled onet.Fatal(err)") + } + } + +} + +func TestAfter(t *testing.T) { + a := lookup.Epoch{ + Time: 5, + Level: 3, + } + b := lookup.Epoch{ + Time: 6, + Level: 3, + } + c := lookup.Epoch{ + Time: 6, + Level: 4, + } + + if !b.After(a) { + t.Fatal("Expected 'after' to be true, got false") + } + + if b.After(b) { + t.Fatal("Expected 'after' to be false when both epochs are identical, got true") + } + + if !b.After(c) { + t.Fatal("Expected 'after' to be true when both epochs have the same time but the level is lower in the first one, but got false") + } + +} diff --git a/swarm/storage/feed/lookup/lookup.go b/swarm/storage/feed/lookup/lookup.go new file mode 100644 index 0000000000..2f862d81c4 --- /dev/null +++ b/swarm/storage/feed/lookup/lookup.go @@ -0,0 +1,180 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Package lookup defines feed lookup algorithms and provides tools to place updates +so they can be found +*/ +package lookup + +const maxuint64 = ^uint64(0) + +// LowestLevel establishes the frequency resolution of the lookup algorithm as a power of 2. +const LowestLevel uint8 = 0 // default is 0 (1 second) + +// HighestLevel sets the lowest frequency the algorithm will operate at, as a power of 2. +// 25 -> 2^25 equals to roughly one year. +const HighestLevel = 25 // default is 25 (~1 year) + +// DefaultLevel sets what level will be chosen to search when there is no hint +const DefaultLevel = HighestLevel + +//Algorithm is the function signature of a lookup algorithm +type Algorithm func(now uint64, hint Epoch, read ReadFunc) (value interface{}, err error) + +// Lookup finds the update with the highest timestamp that is smaller or equal than 'now' +// It takes a hint which should be the epoch where the last known update was +// If you don't know in what epoch the last update happened, simply submit lookup.NoClue +// read() will be called on each lookup attempt +// Returns an error only if read() returns an error +// Returns nil if an update was not found +var Lookup Algorithm = FluzCapacitorAlgorithm + +// ReadFunc is a handler called by Lookup each time it attempts to find a value +// It should return if a value is not found +// It should return if a value is found, but its timestamp is higher than "now" +// It should only return an error in case the handler wants to stop the +// lookup process entirely. +type ReadFunc func(epoch Epoch, now uint64) (interface{}, error) + +// NoClue is a hint that can be provided when the Lookup caller does not have +// a clue about where the last update may be +var NoClue = Epoch{} + +// getBaseTime returns the epoch base time of the given +// time and level +func getBaseTime(t uint64, level uint8) uint64 { + return t & (maxuint64 << level) +} + +// Hint creates a hint based only on the last known update time +func Hint(last uint64) Epoch { + return Epoch{ + Time: last, + Level: DefaultLevel, + } +} + +// GetNextLevel returns the frequency level a next update should be placed at, provided where +// the last update was and what time it is now. +// This is the first nonzero bit of the XOR of 'last' and 'now', counting from the highest significant bit +// but limited to not return a level that is smaller than the last-1 +func GetNextLevel(last Epoch, now uint64) uint8 { + // First XOR the last epoch base time with the current clock. + // This will set all the common most significant bits to zero. + mix := (last.Base() ^ now) + + // Then, make sure we stop the below loop before one level below the current, by setting + // that level's bit to 1. + // If the next level is lower than the current one, it must be exactly level-1 and not lower. + mix |= (1 << (last.Level - 1)) + + // if the last update was more than 2^highestLevel seconds ago, choose the highest level + if mix > (maxuint64 >> (64 - HighestLevel - 1)) { + return HighestLevel + } + + // set up a mask to scan for nonzero bits, starting at the highest level + mask := uint64(1 << (HighestLevel)) + + for i := uint8(HighestLevel); i > LowestLevel; i-- { + if mix&mask != 0 { // if we find a nonzero bit, this is the level the next update should be at. + return i + } + mask = mask >> 1 // move our bit one position to the right + } + return 0 +} + +// GetNextEpoch returns the epoch where the next update should be located +// according to where the previous update was +// and what time it is now. +func GetNextEpoch(last Epoch, now uint64) Epoch { + if last == NoClue { + return GetFirstEpoch(now) + } + level := GetNextLevel(last, now) + return Epoch{ + Level: level, + Time: now, + } +} + +// GetFirstEpoch returns the epoch where the first update should be located +// based on what time it is now. +func GetFirstEpoch(now uint64) Epoch { + return Epoch{Level: HighestLevel, Time: now} +} + +var worstHint = Epoch{Time: 0, Level: 63} + +// FluzCapacitorAlgorithm works by narrowing the epoch search area if an update is found +// going back and forth in time +// First, it will attempt to find an update where it should be now if the hint was +// really the last update. If that lookup fails, then the last update must be either the hint itself +// or the epochs right below. If however, that lookup succeeds, then the update must be +// that one or within the epochs right below. +// see the guide for a more graphical representation +func FluzCapacitorAlgorithm(now uint64, hint Epoch, read ReadFunc) (value interface{}, err error) { + var lastFound interface{} + var epoch Epoch + if hint == NoClue { + hint = worstHint + } + + t := now + + for { + epoch = GetNextEpoch(hint, t) + value, err = read(epoch, now) + if err != nil { + return nil, err + } + if value != nil { + lastFound = value + if epoch.Level == LowestLevel || epoch.Equals(hint) { + return value, nil + } + hint = epoch + continue + } + if epoch.Base() == hint.Base() { + if lastFound != nil { + return lastFound, nil + } + // we have reached the hint itself + if hint == worstHint { + return nil, nil + } + // check it out + value, err = read(hint, now) + if err != nil { + return nil, err + } + if value != nil { + return value, nil + } + // bad hint. + epoch = hint + hint = worstHint + } + base := epoch.Base() + if base == 0 { + return nil, nil + } + t = base - 1 + } +} diff --git a/swarm/storage/feed/lookup/lookup_test.go b/swarm/storage/feed/lookup/lookup_test.go new file mode 100644 index 0000000000..d71e81e95f --- /dev/null +++ b/swarm/storage/feed/lookup/lookup_test.go @@ -0,0 +1,414 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lookup_test + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +type Data struct { + Payload uint64 + Time uint64 +} + +type Store map[lookup.EpochID]*Data + +func write(store Store, epoch lookup.Epoch, value *Data) { + log.Debug("Write: %d-%d, value='%d'\n", epoch.Base(), epoch.Level, value.Payload) + store[epoch.ID()] = value +} + +func update(store Store, last lookup.Epoch, now uint64, value *Data) lookup.Epoch { + epoch := lookup.GetNextEpoch(last, now) + + write(store, epoch, value) + + return epoch +} + +const Day = 60 * 60 * 24 +const Year = Day * 365 +const Month = Day * 30 + +func makeReadFunc(store Store, counter *int) lookup.ReadFunc { + return func(epoch lookup.Epoch, now uint64) (interface{}, error) { + *counter++ + data := store[epoch.ID()] + var valueStr string + if data != nil { + valueStr = fmt.Sprintf("%d", data.Payload) + } + log.Debug("Read: %d-%d, value='%s'\n", epoch.Base(), epoch.Level, valueStr) + if data != nil && data.Time <= now { + return data, nil + } + return nil, nil + } +} + +func TestLookup(t *testing.T) { + + store := make(Store) + readCount := 0 + readFunc := makeReadFunc(store, &readCount) + + // write an update every month for 12 months 3 years ago and then silence for two years + now := uint64(1533799046) + var epoch lookup.Epoch + + var lastData *Data + for i := uint64(0); i < 12; i++ { + t := uint64(now - Year*3 + i*Month) + data := Data{ + Payload: t, //our "payload" will be the timestamp itself. + Time: t, + } + epoch = update(store, epoch, t, &data) + lastData = &data + } + + // try to get the last value + + value, err := lookup.Lookup(now, lookup.NoClue, readFunc) + if err != nil { + t.Fatal(err) + } + + readCountWithoutHint := readCount + + if value != lastData { + t.Fatalf("Expected lookup to return the last written value: %v. Got %v", lastData, value) + } + + // reset the read count for the next test + readCount = 0 + // Provide a hint to get a faster lookup. In particular, we give the exact location of the last update + value, err = lookup.Lookup(now, epoch, readFunc) + if err != nil { + t.Fatal(err) + } + + if value != lastData { + t.Fatalf("Expected lookup to return the last written value: %v. Got %v", lastData, value) + } + + if readCount > readCountWithoutHint { + t.Fatalf("Expected lookup to complete with fewer or same reads than %d since we provided a hint. Did %d reads.", readCountWithoutHint, readCount) + } + + // try to get an intermediate value + // if we look for a value in now - Year*3 + 6*Month, we should get that value + // Since the "payload" is the timestamp itself, we can check this. + + expectedTime := now - Year*3 + 6*Month + + value, err = lookup.Lookup(expectedTime, lookup.NoClue, readFunc) + if err != nil { + t.Fatal(err) + } + + data, ok := value.(*Data) + + if !ok { + t.Fatal("Expected value to contain data") + } + + if data.Time != expectedTime { + t.Fatalf("Expected value timestamp to be %d, got %d", data.Time, expectedTime) + } + +} + +func TestOneUpdateAt0(t *testing.T) { + + store := make(Store) + readCount := 0 + + readFunc := makeReadFunc(store, &readCount) + now := uint64(1533903729) + + var epoch lookup.Epoch + data := Data{ + Payload: 79, + Time: 0, + } + update(store, epoch, 0, &data) + + value, err := lookup.Lookup(now, lookup.NoClue, readFunc) + if err != nil { + t.Fatal(err) + } + if value != &data { + t.Fatalf("Expected lookup to return the last written value: %v. Got %v", data, value) + } +} + +// Tests the update is found even when a bad hint is given +func TestBadHint(t *testing.T) { + + store := make(Store) + readCount := 0 + + readFunc := makeReadFunc(store, &readCount) + now := uint64(1533903729) + + var epoch lookup.Epoch + data := Data{ + Payload: 79, + Time: 0, + } + + // place an update for t=1200 + update(store, epoch, 1200, &data) + + // come up with some evil hint + badHint := lookup.Epoch{ + Level: 18, + Time: 1200000000, + } + + value, err := lookup.Lookup(now, badHint, readFunc) + if err != nil { + t.Fatal(err) + } + if value != &data { + t.Fatalf("Expected lookup to return the last written value: %v. Got %v", data, value) + } +} + +func TestLookupFail(t *testing.T) { + + store := make(Store) + readCount := 0 + + readFunc := makeReadFunc(store, &readCount) + now := uint64(1533903729) + + // don't write anything and try to look up. + // we're testing we don't get stuck in a loop + + value, err := lookup.Lookup(now, lookup.NoClue, readFunc) + if err != nil { + t.Fatal(err) + } + if value != nil { + t.Fatal("Expected value to be nil, since the update should've failed") + } + + expectedReads := now/(1< readCountWithoutHint { + t.Fatalf("Expected lookup to complete with fewer or equal reads than %d since we provided a hint. Did %d reads.", readCountWithoutHint, readCount) + } + + for i := uint64(0); i <= 994; i++ { + T := uint64(now - 1000 + i) // update every second for the last 1000 seconds + value, err := lookup.Lookup(T, lookup.NoClue, readFunc) + if err != nil { + t.Fatal(err) + } + data, _ := value.(*Data) + if data == nil { + t.Fatalf("Expected lookup to return %d, got nil", T) + } + if data.Payload != T { + t.Fatalf("Expected lookup to return %d, got %d", T, data.Time) + } + } +} + +func TestSparseUpdates(t *testing.T) { + + store := make(Store) + readCount := 0 + readFunc := makeReadFunc(store, &readCount) + + // write an update every 5 years 3 times starting in Jan 1st 1970 and then silence + + now := uint64(1533799046) + var epoch lookup.Epoch + + var lastData *Data + for i := uint64(0); i < 5; i++ { + T := uint64(Year * 5 * i) // write an update every 5 years 3 times starting in Jan 1st 1970 and then silence + data := Data{ + Payload: T, //our "payload" will be the timestamp itself. + Time: T, + } + epoch = update(store, epoch, T, &data) + lastData = &data + } + + // try to get the last value + + value, err := lookup.Lookup(now, lookup.NoClue, readFunc) + if err != nil { + t.Fatal(err) + } + + readCountWithoutHint := readCount + + if value != lastData { + t.Fatalf("Expected lookup to return the last written value: %v. Got %v", lastData, value) + } + + // reset the read count for the next test + readCount = 0 + // Provide a hint to get a faster lookup. In particular, we give the exact location of the last update + value, err = lookup.Lookup(now, epoch, readFunc) + if err != nil { + t.Fatal(err) + } + + if value != lastData { + t.Fatalf("Expected lookup to return the last written value: %v. Got %v", lastData, value) + } + + if readCount > readCountWithoutHint { + t.Fatalf("Expected lookup to complete with fewer reads than %d since we provided a hint. Did %d reads.", readCountWithoutHint, readCount) + } + +} + +// testG will hold precooked test results +// fields are abbreviated to reduce the size of the literal below +type testG struct { + e lookup.Epoch // last + n uint64 // next level + x uint8 // expected result +} + +// test cases +var testGetNextLevelCases []testG = []testG{{e: lookup.Epoch{Time: 989875233, Level: 12}, n: 989875233, x: 11}, {e: lookup.Epoch{Time: 995807650, Level: 18}, n: 995598156, x: 19}, {e: lookup.Epoch{Time: 969167082, Level: 0}, n: 968990357, x: 18}, {e: lookup.Epoch{Time: 993087628, Level: 14}, n: 992987044, x: 20}, {e: lookup.Epoch{Time: 963364631, Level: 20}, n: 963364630, x: 19}, {e: lookup.Epoch{Time: 963497510, Level: 16}, n: 963370732, x: 18}, {e: lookup.Epoch{Time: 955421349, Level: 22}, n: 955421348, x: 21}, {e: lookup.Epoch{Time: 968220379, Level: 15}, n: 968220378, x: 14}, {e: lookup.Epoch{Time: 939129014, Level: 6}, n: 939128771, x: 11}, {e: lookup.Epoch{Time: 907847903, Level: 6}, n: 907791833, x: 18}, {e: lookup.Epoch{Time: 910835564, Level: 15}, n: 910835564, x: 14}, {e: lookup.Epoch{Time: 913578333, Level: 22}, n: 881808431, x: 25}, {e: lookup.Epoch{Time: 895818460, Level: 3}, n: 895818132, x: 9}, {e: lookup.Epoch{Time: 903843025, Level: 24}, n: 895609561, x: 23}, {e: lookup.Epoch{Time: 877889433, Level: 13}, n: 877877093, x: 15}, {e: lookup.Epoch{Time: 901450396, Level: 10}, n: 901450058, x: 9}, {e: lookup.Epoch{Time: 925179910, Level: 3}, n: 925168393, x: 16}, {e: lookup.Epoch{Time: 913485477, Level: 21}, n: 913485476, x: 20}, {e: lookup.Epoch{Time: 924462991, Level: 18}, n: 924462990, x: 17}, {e: lookup.Epoch{Time: 941175128, Level: 13}, n: 941175127, x: 12}, {e: lookup.Epoch{Time: 920126583, Level: 3}, n: 920100782, x: 19}, {e: lookup.Epoch{Time: 932403200, Level: 9}, n: 932279891, x: 17}, {e: lookup.Epoch{Time: 948284931, Level: 2}, n: 948284921, x: 9}, {e: lookup.Epoch{Time: 953540997, Level: 7}, n: 950547986, x: 22}, {e: lookup.Epoch{Time: 926639837, Level: 18}, n: 918608882, x: 24}, {e: lookup.Epoch{Time: 954637598, Level: 1}, n: 954578761, x: 17}, {e: lookup.Epoch{Time: 943482981, Level: 10}, n: 942924151, x: 19}, {e: lookup.Epoch{Time: 963580771, Level: 7}, n: 963580771, x: 6}, {e: lookup.Epoch{Time: 993744930, Level: 7}, n: 993690858, x: 16}, {e: lookup.Epoch{Time: 1018890213, Level: 12}, n: 1018890212, x: 11}, {e: lookup.Epoch{Time: 1030309411, Level: 2}, n: 1030309227, x: 9}, {e: lookup.Epoch{Time: 1063204997, Level: 20}, n: 1063204996, x: 19}, {e: lookup.Epoch{Time: 1094340832, Level: 6}, n: 1094340633, x: 7}, {e: lookup.Epoch{Time: 1077880597, Level: 10}, n: 1075914292, x: 20}, {e: lookup.Epoch{Time: 1051114957, Level: 18}, n: 1051114957, x: 17}, {e: lookup.Epoch{Time: 1045649701, Level: 22}, n: 1045649700, x: 21}, {e: lookup.Epoch{Time: 1066198885, Level: 14}, n: 1066198884, x: 13}, {e: lookup.Epoch{Time: 1053231952, Level: 1}, n: 1053210845, x: 16}, {e: lookup.Epoch{Time: 1068763404, Level: 14}, n: 1068675428, x: 18}, {e: lookup.Epoch{Time: 1039042173, Level: 15}, n: 1038973110, x: 17}, {e: lookup.Epoch{Time: 1050747636, Level: 6}, n: 1050747364, x: 9}, {e: lookup.Epoch{Time: 1030034434, Level: 23}, n: 1030034433, x: 22}, {e: lookup.Epoch{Time: 1003783425, Level: 18}, n: 1003783424, x: 17}, {e: lookup.Epoch{Time: 988163976, Level: 15}, n: 988084064, x: 17}, {e: lookup.Epoch{Time: 1007222377, Level: 15}, n: 1007222377, x: 14}, {e: lookup.Epoch{Time: 1001211375, Level: 13}, n: 1001208178, x: 14}, {e: lookup.Epoch{Time: 997623199, Level: 8}, n: 997623198, x: 7}, {e: lookup.Epoch{Time: 1026283830, Level: 10}, n: 1006681704, x: 24}, {e: lookup.Epoch{Time: 1019421907, Level: 20}, n: 1019421906, x: 19}, {e: lookup.Epoch{Time: 1043154306, Level: 16}, n: 1043108343, x: 16}, {e: lookup.Epoch{Time: 1075643767, Level: 17}, n: 1075325898, x: 18}, {e: lookup.Epoch{Time: 1043726309, Level: 20}, n: 1043726308, x: 19}, {e: lookup.Epoch{Time: 1056415324, Level: 17}, n: 1056415324, x: 16}, {e: lookup.Epoch{Time: 1088650219, Level: 13}, n: 1088650218, x: 12}, {e: lookup.Epoch{Time: 1088551662, Level: 7}, n: 1088543355, x: 13}, {e: lookup.Epoch{Time: 1069667265, Level: 6}, n: 1069667075, x: 7}, {e: lookup.Epoch{Time: 1079145970, Level: 18}, n: 1079145969, x: 17}, {e: lookup.Epoch{Time: 1083338876, Level: 7}, n: 1083338875, x: 6}, {e: lookup.Epoch{Time: 1051581086, Level: 4}, n: 1051568869, x: 14}, {e: lookup.Epoch{Time: 1028430882, Level: 4}, n: 1028430864, x: 5}, {e: lookup.Epoch{Time: 1057356462, Level: 1}, n: 1057356417, x: 5}, {e: lookup.Epoch{Time: 1033104266, Level: 0}, n: 1033097479, x: 13}, {e: lookup.Epoch{Time: 1031391367, Level: 11}, n: 1031387304, x: 14}, {e: lookup.Epoch{Time: 1049781164, Level: 15}, n: 1049781163, x: 14}, {e: lookup.Epoch{Time: 1027271628, Level: 12}, n: 1027271627, x: 11}, {e: lookup.Epoch{Time: 1057270560, Level: 23}, n: 1057270560, x: 22}, {e: lookup.Epoch{Time: 1047501317, Level: 15}, n: 1047501317, x: 14}, {e: lookup.Epoch{Time: 1058349035, Level: 11}, n: 1045175573, x: 24}, {e: lookup.Epoch{Time: 1057396147, Level: 20}, n: 1057396147, x: 19}, {e: lookup.Epoch{Time: 1048906375, Level: 18}, n: 1039616919, x: 25}, {e: lookup.Epoch{Time: 1074294831, Level: 20}, n: 1074294831, x: 19}, {e: lookup.Epoch{Time: 1088946052, Level: 1}, n: 1088917364, x: 14}, {e: lookup.Epoch{Time: 1112337595, Level: 17}, n: 1111008110, x: 22}, {e: lookup.Epoch{Time: 1099990284, Level: 5}, n: 1099968370, x: 15}, {e: lookup.Epoch{Time: 1087036441, Level: 16}, n: 1053967855, x: 25}, {e: lookup.Epoch{Time: 1069225185, Level: 8}, n: 1069224660, x: 10}, {e: lookup.Epoch{Time: 1057505479, Level: 9}, n: 1057505170, x: 14}, {e: lookup.Epoch{Time: 1072381377, Level: 12}, n: 1065950959, x: 22}, {e: lookup.Epoch{Time: 1093887139, Level: 8}, n: 1093863305, x: 14}, {e: lookup.Epoch{Time: 1082366510, Level: 24}, n: 1082366510, x: 23}, {e: lookup.Epoch{Time: 1103231132, Level: 14}, n: 1102292201, x: 22}, {e: lookup.Epoch{Time: 1094502355, Level: 3}, n: 1094324652, x: 18}, {e: lookup.Epoch{Time: 1068488344, Level: 12}, n: 1067577330, x: 19}, {e: lookup.Epoch{Time: 1050278233, Level: 12}, n: 1050278232, x: 11}, {e: lookup.Epoch{Time: 1047660768, Level: 5}, n: 1047652137, x: 17}, {e: lookup.Epoch{Time: 1060116167, Level: 11}, n: 1060114091, x: 12}, {e: lookup.Epoch{Time: 1068149392, Level: 21}, n: 1052074801, x: 24}, {e: lookup.Epoch{Time: 1081934120, Level: 6}, n: 1081933847, x: 8}, {e: lookup.Epoch{Time: 1107943693, Level: 16}, n: 1107096139, x: 25}, {e: lookup.Epoch{Time: 1131571649, Level: 9}, n: 1131570428, x: 11}, {e: lookup.Epoch{Time: 1123139367, Level: 0}, n: 1122912198, x: 20}, {e: lookup.Epoch{Time: 1121144423, Level: 6}, n: 1120568289, x: 20}, {e: lookup.Epoch{Time: 1089932411, Level: 17}, n: 1089932410, x: 16}, {e: lookup.Epoch{Time: 1104899012, Level: 22}, n: 1098978789, x: 22}, {e: lookup.Epoch{Time: 1094588059, Level: 21}, n: 1094588059, x: 20}, {e: lookup.Epoch{Time: 1114987438, Level: 24}, n: 1114987437, x: 23}, {e: lookup.Epoch{Time: 1084186305, Level: 7}, n: 1084186241, x: 6}, {e: lookup.Epoch{Time: 1058827111, Level: 8}, n: 1058826504, x: 9}, {e: lookup.Epoch{Time: 1090679810, Level: 12}, n: 1090616539, x: 17}, {e: lookup.Epoch{Time: 1084299475, Level: 23}, n: 1084299475, x: 22}} + +func TestGetNextLevel(t *testing.T) { + + // First, test well-known cases + last := lookup.Epoch{ + Time: 1533799046, + Level: 5, + } + + level := lookup.GetNextLevel(last, last.Time) + expected := uint8(4) + if level != expected { + t.Fatalf("Expected GetNextLevel to return %d for same-time updates at a nonzero level, got %d", expected, level) + } + + level = lookup.GetNextLevel(last, last.Time+(1< lookup.HighestLevel { + v = 0 + } + now = last.Time + uint64(rand.Intn(1<. + +package feed + +import ( + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +// Query is used to specify constraints when performing an update lookup +// TimeLimit indicates an upper bound for the search. Set to 0 for "now" +type Query struct { + Feed + Hint lookup.Epoch + TimeLimit uint64 +} + +// FromValues deserializes this instance from a string key-value store +// useful to parse query strings +func (q *Query) FromValues(values Values) error { + time, _ := strconv.ParseUint(values.Get("time"), 10, 64) + q.TimeLimit = time + + level, _ := strconv.ParseUint(values.Get("hint.level"), 10, 32) + q.Hint.Level = uint8(level) + q.Hint.Time, _ = strconv.ParseUint(values.Get("hint.time"), 10, 64) + if q.Feed.User == (common.Address{}) { + return q.Feed.FromValues(values) + } + return nil +} + +// AppendValues serializes this structure into the provided string key-value store +// useful to build query strings +func (q *Query) AppendValues(values Values) { + if q.TimeLimit != 0 { + values.Set("time", fmt.Sprintf("%d", q.TimeLimit)) + } + if q.Hint.Level != 0 { + values.Set("hint.level", fmt.Sprintf("%d", q.Hint.Level)) + } + if q.Hint.Time != 0 { + values.Set("hint.time", fmt.Sprintf("%d", q.Hint.Time)) + } + q.Feed.AppendValues(values) +} + +// NewQuery constructs an Query structure to find updates on or before `time` +// if time == 0, the latest update will be looked up +func NewQuery(feed *Feed, time uint64, hint lookup.Epoch) *Query { + return &Query{ + TimeLimit: time, + Feed: *feed, + Hint: hint, + } +} + +// NewQueryLatest generates lookup parameters that look for the latest update to a feed +func NewQueryLatest(feed *Feed, hint lookup.Epoch) *Query { + return NewQuery(feed, 0, hint) +} diff --git a/swarm/storage/feed/query_test.go b/swarm/storage/feed/query_test.go new file mode 100644 index 0000000000..9fa5e29800 --- /dev/null +++ b/swarm/storage/feed/query_test.go @@ -0,0 +1,38 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "testing" +) + +func getTestQuery() *Query { + id := getTestID() + return &Query{ + TimeLimit: 5000, + Feed: id.Feed, + Hint: id.Epoch, + } +} + +func TestQueryValues(t *testing.T) { + var expected = KV{"hint.level": "25", "hint.time": "1000", "time": "5000", "topic": "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000", "user": "0x876A8936A7Cd0b79Ef0735AD0896c1AFe278781c"} + + query := getTestQuery() + testValueSerializer(t, query, expected) + +} diff --git a/swarm/storage/feed/request.go b/swarm/storage/feed/request.go new file mode 100644 index 0000000000..6968d8b9ad --- /dev/null +++ b/swarm/storage/feed/request.go @@ -0,0 +1,284 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "bytes" + "encoding/json" + "hash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +// Request represents a request to sign or signed feed update message +type Request struct { + Update // actual content that will be put on the chunk, less signature + Signature *Signature + idAddr storage.Address // cached chunk address for the update (not serialized, for internal use) + binaryData []byte // cached serialized data (does not get serialized again!, for efficiency/internal use) +} + +// updateRequestJSON represents a JSON-serialized UpdateRequest +type updateRequestJSON struct { + ID + ProtocolVersion uint8 `json:"protocolVersion"` + Data string `json:"data,omitempty"` + Signature string `json:"signature,omitempty"` +} + +// Request layout +// Update bytes +// SignatureLength bytes +const minimumSignedUpdateLength = minimumUpdateDataLength + signatureLength + +// NewFirstRequest returns a ready to sign request to publish a first feed update +func NewFirstRequest(topic Topic) *Request { + + request := new(Request) + + // get the current time + now := TimestampProvider.Now().Time + request.Epoch = lookup.GetFirstEpoch(now) + request.Feed.Topic = topic + request.Header.Version = ProtocolVersion + + return request +} + +// SetData stores the payload data the feed update will be updated with +func (r *Request) SetData(data []byte) { + r.data = data + r.Signature = nil +} + +// IsUpdate returns true if this request models a signed update or otherwise it is a signature request +func (r *Request) IsUpdate() bool { + return r.Signature != nil +} + +// Verify checks that signatures are valid +func (r *Request) Verify() (err error) { + if len(r.data) == 0 { + return NewError(ErrInvalidValue, "Update does not contain data") + } + if r.Signature == nil { + return NewError(ErrInvalidSignature, "Missing signature field") + } + + digest, err := r.GetDigest() + if err != nil { + return err + } + + // get the address of the signer (which also checks that it's a valid signature) + r.Feed.User, err = getUserAddr(digest, *r.Signature) + if err != nil { + return err + } + + // check that the lookup information contained in the chunk matches the updateAddr (chunk search key) + // that was used to retrieve this chunk + // if this validation fails, someone forged a chunk. + if !bytes.Equal(r.idAddr, r.Addr()) { + return NewError(ErrInvalidSignature, "Signature address does not match with update user address") + } + + return nil +} + +// Sign executes the signature to validate the update message +func (r *Request) Sign(signer Signer) error { + r.Feed.User = signer.Address() + r.binaryData = nil //invalidate serialized data + digest, err := r.GetDigest() // computes digest and serializes into .binaryData + if err != nil { + return err + } + + signature, err := signer.Sign(digest) + if err != nil { + return err + } + + // Although the Signer interface returns the public address of the signer, + // recover it from the signature to see if they match + userAddr, err := getUserAddr(digest, signature) + if err != nil { + return NewError(ErrInvalidSignature, "Error verifying signature") + } + + if userAddr != signer.Address() { // sanity check to make sure the Signer is declaring the same address used to sign! + return NewError(ErrInvalidSignature, "Signer address does not match update user address") + } + + r.Signature = &signature + r.idAddr = r.Addr() + return nil +} + +// GetDigest creates the feed update digest used in signatures +// the serialized payload is cached in .binaryData +func (r *Request) GetDigest() (result common.Hash, err error) { + hasher := hashPool.Get().(hash.Hash) + defer hashPool.Put(hasher) + hasher.Reset() + dataLength := r.Update.binaryLength() + if r.binaryData == nil { + r.binaryData = make([]byte, dataLength+signatureLength) + if err := r.Update.binaryPut(r.binaryData[:dataLength]); err != nil { + return result, err + } + } + hasher.Write(r.binaryData[:dataLength]) //everything except the signature. + + return common.BytesToHash(hasher.Sum(nil)), nil +} + +// create an update chunk. +func (r *Request) toChunk() (storage.Chunk, error) { + + // Check that the update is signed and serialized + // For efficiency, data is serialized during signature and cached in + // the binaryData field when computing the signature digest in .getDigest() + if r.Signature == nil || r.binaryData == nil { + return nil, NewError(ErrInvalidSignature, "toChunk called without a valid signature or payload data. Call .Sign() first.") + } + + updateLength := r.Update.binaryLength() + + // signature is the last item in the chunk data + copy(r.binaryData[updateLength:], r.Signature[:]) + + chunk := storage.NewChunk(r.idAddr, r.binaryData) + return chunk, nil +} + +// fromChunk populates this structure from chunk data. It does not verify the signature is valid. +func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error { + // for update chunk layout see Request definition + + //deserialize the feed update portion + if err := r.Update.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil { + return err + } + + // Extract the signature + var signature *Signature + cursor := r.Update.binaryLength() + sigdata := chunkdata[cursor : cursor+signatureLength] + if len(sigdata) > 0 { + signature = &Signature{} + copy(signature[:], sigdata) + } + + r.Signature = signature + r.idAddr = updateAddr + r.binaryData = chunkdata + + return nil + +} + +// FromValues deserializes this instance from a string key-value store +// useful to parse query strings +func (r *Request) FromValues(values Values, data []byte) error { + signatureBytes, err := hexutil.Decode(values.Get("signature")) + if err != nil { + r.Signature = nil + } else { + if len(signatureBytes) != signatureLength { + return NewError(ErrInvalidSignature, "Incorrect signature length") + } + r.Signature = new(Signature) + copy(r.Signature[:], signatureBytes) + } + err = r.Update.FromValues(values, data) + if err != nil { + return err + } + r.idAddr = r.Addr() + return err +} + +// AppendValues serializes this structure into the provided string key-value store +// useful to build query strings +func (r *Request) AppendValues(values Values) []byte { + if r.Signature != nil { + values.Set("signature", hexutil.Encode(r.Signature[:])) + } + return r.Update.AppendValues(values) +} + +// fromJSON takes an update request JSON and populates an UpdateRequest +func (r *Request) fromJSON(j *updateRequestJSON) error { + + r.ID = j.ID + r.Header.Version = j.ProtocolVersion + + var err error + if j.Data != "" { + r.data, err = hexutil.Decode(j.Data) + if err != nil { + return NewError(ErrInvalidValue, "Cannot decode data") + } + } + + if j.Signature != "" { + sigBytes, err := hexutil.Decode(j.Signature) + if err != nil || len(sigBytes) != signatureLength { + return NewError(ErrInvalidSignature, "Cannot decode signature") + } + r.Signature = new(Signature) + r.idAddr = r.Addr() + copy(r.Signature[:], sigBytes) + } + return nil +} + +// UnmarshalJSON takes a JSON structure stored in a byte array and populates the Request object +// Implements json.Unmarshaler interface +func (r *Request) UnmarshalJSON(rawData []byte) error { + var requestJSON updateRequestJSON + if err := json.Unmarshal(rawData, &requestJSON); err != nil { + return err + } + return r.fromJSON(&requestJSON) +} + +// MarshalJSON takes an update request and encodes it as a JSON structure into a byte array +// Implements json.Marshaler interface +func (r *Request) MarshalJSON() (rawData []byte, err error) { + var signatureString, dataString string + if r.Signature != nil { + signatureString = hexutil.Encode(r.Signature[:]) + } + if r.data != nil { + dataString = hexutil.Encode(r.data) + } + + requestJSON := &updateRequestJSON{ + ID: r.ID, + ProtocolVersion: r.Header.Version, + Data: dataString, + Signature: signatureString, + } + + return json.Marshal(requestJSON) +} diff --git a/swarm/storage/feed/request_test.go b/swarm/storage/feed/request_test.go new file mode 100644 index 0000000000..f5de32b74f --- /dev/null +++ b/swarm/storage/feed/request_test.go @@ -0,0 +1,312 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" +) + +func areEqualJSON(s1, s2 string) (bool, error) { + //credit for the trick: turtlemonvh https://gist.github.com/turtlemonvh/e4f7404e28387fadb8ad275a99596f67 + var o1 interface{} + var o2 interface{} + + err := json.Unmarshal([]byte(s1), &o1) + if err != nil { + return false, fmt.Errorf("Error mashalling string 1 :: %s", err.Error()) + } + err = json.Unmarshal([]byte(s2), &o2) + if err != nil { + return false, fmt.Errorf("Error mashalling string 2 :: %s", err.Error()) + } + + return reflect.DeepEqual(o1, o2), nil +} + +// TestEncodingDecodingUpdateRequests ensures that requests are serialized properly +// while also checking cryptographically that only the owner of a feed can update it. +func TestEncodingDecodingUpdateRequests(t *testing.T) { + + charlie := newCharlieSigner() //Charlie + bob := newBobSigner() //Bob + + // Create a feed to our good guy Charlie's name + topic, _ := NewTopic("a good topic name", nil) + firstRequest := NewFirstRequest(topic) + firstRequest.User = charlie.Address() + + // We now encode the create message to simulate we send it over the wire + messageRawData, err := firstRequest.MarshalJSON() + if err != nil { + t.Fatalf("Error encoding first feed update request: %s", err) + } + + // ... the message arrives and is decoded... + var recoveredFirstRequest Request + if err := recoveredFirstRequest.UnmarshalJSON(messageRawData); err != nil { + t.Fatalf("Error decoding first feed update request: %s", err) + } + + // ... but verification should fail because it is not signed! + if err := recoveredFirstRequest.Verify(); err == nil { + t.Fatal("Expected Verify to fail since the message is not signed") + } + + // We now assume that the feed ypdate was created and propagated. + + const expectedSignature = "0x7235b27a68372ddebcf78eba48543fa460864b0b0e99cb533fcd3664820e603312d29426dd00fb39628f5299480a69bf6e462838d78de49ce0704c754c9deb2601" + const expectedJSON = `{"feed":{"topic":"0x6120676f6f6420746f706963206e616d65000000000000000000000000000000","user":"0x876a8936a7cd0b79ef0735ad0896c1afe278781c"},"epoch":{"time":1000,"level":1},"protocolVersion":0,"data":"0x5468697320686f75722773207570646174653a20537761726d2039392e3020686173206265656e2072656c656173656421"}` + + //Put together an unsigned update request that we will serialize to send it to the signer. + data := []byte("This hour's update: Swarm 99.0 has been released!") + request := &Request{ + Update: Update{ + ID: ID{ + Epoch: lookup.Epoch{ + Time: 1000, + Level: 1, + }, + Feed: firstRequest.Update.Feed, + }, + data: data, + }, + } + + messageRawData, err = request.MarshalJSON() + if err != nil { + t.Fatalf("Error encoding update request: %s", err) + } + + equalJSON, err := areEqualJSON(string(messageRawData), expectedJSON) + if err != nil { + t.Fatalf("Error decoding update request JSON: %s", err) + } + if !equalJSON { + t.Fatalf("Received a different JSON message. Expected %s, got %s", expectedJSON, string(messageRawData)) + } + + // now the encoded message messageRawData is sent over the wire and arrives to the signer + + //Attempt to extract an UpdateRequest out of the encoded message + var recoveredRequest Request + if err := recoveredRequest.UnmarshalJSON(messageRawData); err != nil { + t.Fatalf("Error decoding update request: %s", err) + } + + //sign the request and see if it matches our predefined signature above. + if err := recoveredRequest.Sign(charlie); err != nil { + t.Fatalf("Error signing request: %s", err) + } + + compareByteSliceToExpectedHex(t, "signature", recoveredRequest.Signature[:], expectedSignature) + + // mess with the signature and see what happens. To alter the signature, we briefly decode it as JSON + // to alter the signature field. + var j updateRequestJSON + if err := json.Unmarshal([]byte(expectedJSON), &j); err != nil { + t.Fatal("Error unmarshalling test json, check expectedJSON constant") + } + j.Signature = "Certainly not a signature" + corruptMessage, _ := json.Marshal(j) // encode the message with the bad signature + var corruptRequest Request + if err = corruptRequest.UnmarshalJSON(corruptMessage); err == nil { + t.Fatal("Expected DecodeUpdateRequest to fail when trying to interpret a corrupt message with an invalid signature") + } + + // Now imagine Bob wants to create an update of his own about the same feed, + // signing a message with his private key + if err := request.Sign(bob); err != nil { + t.Fatalf("Error signing: %s", err) + } + + // Now Bob encodes the message to send it over the wire... + messageRawData, err = request.MarshalJSON() + if err != nil { + t.Fatalf("Error encoding message:%s", err) + } + + // ... the message arrives to our Swarm node and it is decoded. + recoveredRequest = Request{} + if err := recoveredRequest.UnmarshalJSON(messageRawData); err != nil { + t.Fatalf("Error decoding message:%s", err) + } + + // Before checking what happened with Bob's update, let's see what would happen if we mess + // with the signature big time to see if Verify catches it + savedSignature := *recoveredRequest.Signature // save the signature for later + binary.LittleEndian.PutUint64(recoveredRequest.Signature[5:], 556845463424) // write some random data to break the signature + if err = recoveredRequest.Verify(); err == nil { + t.Fatal("Expected Verify to fail on corrupt signature") + } + + // restore the Bob's signature from corruption + *recoveredRequest.Signature = savedSignature + + // Now the signature is not corrupt + if err = recoveredRequest.Verify(); err != nil { + t.Fatal(err) + } + + // Reuse object and sign with our friend Charlie's private key + if err := recoveredRequest.Sign(charlie); err != nil { + t.Fatalf("Error signing with the correct private key: %s", err) + } + + // And now, Verify should work since this update now belongs to Charlie + if err = recoveredRequest.Verify(); err != nil { + t.Fatalf("Error verifying that Charlie, can sign a reused request object:%s", err) + } + + // mess with the lookup key to make sure Verify fails: + recoveredRequest.Time = 77999 // this will alter the lookup key + if err = recoveredRequest.Verify(); err == nil { + t.Fatalf("Expected Verify to fail since the lookup key has been altered") + } +} + +func getTestRequest() *Request { + return &Request{ + Update: *getTestFeedUpdate(), + } +} + +func TestUpdateChunkSerializationErrorChecking(t *testing.T) { + + // Test that parseUpdate fails if the chunk is too small + var r Request + if err := r.fromChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength)); err == nil { + t.Fatalf("Expected request.fromChunk to fail when chunkData contains less than %d bytes", minimumUpdateDataLength) + } + + r = *getTestRequest() + + _, err := r.toChunk() + if err == nil { + t.Fatal("Expected request.toChunk to fail when there is no data") + } + r.data = []byte("Al bien hacer jamás le falta premio") // put some arbitrary length data + _, err = r.toChunk() + if err == nil { + t.Fatal("expected request.toChunk to fail when there is no signature") + } + + charlie := newCharlieSigner() + if err := r.Sign(charlie); err != nil { + t.Fatalf("error signing:%s", err) + } + + chunk, err := r.toChunk() + if err != nil { + t.Fatalf("error creating update chunk:%s", err) + } + + compareByteSliceToExpectedHex(t, "chunk", chunk.Data(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019416c206269656e206861636572206a616dc3a173206c652066616c7461207072656d696f5a0ffe0bc27f207cd5b00944c8b9cee93e08b89b5ada777f123ac535189333f174a6a4ca2f43a92c4a477a49d774813c36ce8288552c58e6205b0ac35d0507eb00") + + var recovered Request + recovered.fromChunk(chunk.Address(), chunk.Data()) + if !reflect.DeepEqual(recovered, r) { + t.Fatal("Expected recovered feed update request to equal the original one") + } +} + +// check that signature address matches update signer address +func TestReverse(t *testing.T) { + + epoch := lookup.Epoch{ + Time: 7888, + Level: 6, + } + + // make fake timeProvider + timeProvider := &fakeTimeProvider{ + currentTime: startTime.Time, + } + + // signer containing private key + signer := newAliceSigner() + + // set up rpc and create feeds handler + _, _, teardownTest, err := setupTest(timeProvider, signer) + if err != nil { + t.Fatal(err) + } + defer teardownTest() + + topic, _ := NewTopic("Cervantes quotes", nil) + fd := Feed{ + Topic: topic, + User: signer.Address(), + } + + data := []byte("Donde una puerta se cierra, otra se abre") + + request := new(Request) + request.Feed = fd + request.Epoch = epoch + request.data = data + + // generate a chunk key for this request + key := request.Addr() + + if err = request.Sign(signer); err != nil { + t.Fatal(err) + } + + chunk, err := request.toChunk() + if err != nil { + t.Fatal(err) + } + + // check that we can recover the owner account from the update chunk's signature + var checkUpdate Request + if err := checkUpdate.fromChunk(chunk.Address(), chunk.Data()); err != nil { + t.Fatal(err) + } + checkdigest, err := checkUpdate.GetDigest() + if err != nil { + t.Fatal(err) + } + recoveredAddr, err := getUserAddr(checkdigest, *checkUpdate.Signature) + if err != nil { + t.Fatalf("Retrieve address from signature fail: %v", err) + } + originalAddr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) + + // check that the metadata retrieved from the chunk matches what we gave it + if recoveredAddr != originalAddr { + t.Fatalf("addresses dont match: %x != %x", originalAddr, recoveredAddr) + } + + if !bytes.Equal(key[:], chunk.Address()[:]) { + t.Fatalf("Expected chunk key '%x', was '%x'", key, chunk.Address()) + } + if epoch != checkUpdate.Epoch { + t.Fatalf("Expected epoch to be '%s', was '%s'", epoch.String(), checkUpdate.Epoch.String()) + } + if !bytes.Equal(data, checkUpdate.data) { + t.Fatalf("Expected data '%x', was '%x'", data, checkUpdate.data) + } +} diff --git a/swarm/storage/mru/resource_sign.go b/swarm/storage/feed/sign.go similarity index 82% rename from swarm/storage/mru/resource_sign.go rename to swarm/storage/feed/sign.go index a9f7cb6296..5f0ea0b33c 100644 --- a/swarm/storage/mru/resource_sign.go +++ b/swarm/storage/feed/sign.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package mru +package feed import ( "crypto/ecdsa" @@ -28,7 +28,7 @@ const signatureLength = 65 // Signature is an alias for a static byte array with the size of a signature type Signature [signatureLength]byte -// Signer signs Mutable Resource update payloads +// Signer signs feed update payloads type Signer interface { Sign(common.Hash) (Signature, error) Address() common.Address @@ -60,7 +60,16 @@ func (s *GenericSigner) Sign(data common.Hash) (signature Signature, err error) return } -// PublicKey returns the public key of the signer's private key +// Address returns the public key of the signer's private key func (s *GenericSigner) Address() common.Address { return s.address } + +// getUserAddr extracts the address of the feed update signer +func getUserAddr(digest common.Hash, signature Signature) (common.Address, error) { + pub, err := crypto.SigToPub(digest.Bytes(), signature[:]) + if err != nil { + return common.Address{}, err + } + return crypto.PubkeyToAddress(*pub), nil +} diff --git a/swarm/storage/mru/testutil.go b/swarm/storage/feed/testutil.go similarity index 85% rename from swarm/storage/mru/testutil.go rename to swarm/storage/feed/testutil.go index 936132d401..b513fa1f2f 100644 --- a/swarm/storage/mru/testutil.go +++ b/swarm/storage/feed/testutil.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package mru +package feed import ( "context" @@ -27,7 +27,7 @@ import ( ) const ( - testDbDirName = "mru" + testDbDirName = "feeds" ) type TestHandler struct { @@ -40,7 +40,7 @@ func (t *TestHandler) Close() { type mockNetFetcher struct{} -func (m *mockNetFetcher) Request(ctx context.Context) { +func (m *mockNetFetcher) Request(ctx context.Context, hopCount uint8) { } func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { } @@ -52,20 +52,20 @@ func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetF // NewTestHandler creates Handler object to be used for testing purposes. func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) { path := filepath.Join(datadir, testDbDirName) - rh := NewHandler(params) + fh := NewHandler(params) localstoreparams := storage.NewDefaultLocalStoreParams() localstoreparams.Init(path) localStore, err := storage.NewLocalStore(localstoreparams, nil) if err != nil { return nil, fmt.Errorf("localstore create fail, path %s: %v", path, err) } - localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(resourceHashAlgorithm))) - localStore.Validators = append(localStore.Validators, rh) + localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(feedsHashAlgorithm))) + localStore.Validators = append(localStore.Validators, fh) netStore, err := storage.NewNetStore(localStore, nil) if err != nil { return nil, err } netStore.NewNetFetcherFunc = newFakeNetFetcher - rh.SetStore(netStore) - return &TestHandler{rh}, nil + fh.SetStore(netStore) + return &TestHandler{fh}, nil } diff --git a/swarm/storage/mru/timestampprovider.go b/swarm/storage/feed/timestampprovider.go similarity index 78% rename from swarm/storage/mru/timestampprovider.go rename to swarm/storage/feed/timestampprovider.go index f483491aa1..072dc3a48d 100644 --- a/swarm/storage/mru/timestampprovider.go +++ b/swarm/storage/feed/timestampprovider.go @@ -14,19 +14,20 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package mru +package feed import ( "encoding/binary" + "encoding/json" "time" ) -// TimestampProvider sets the time source of the mru package +// TimestampProvider sets the time source of the feeds package var TimestampProvider timestampProvider = NewDefaultTimestampProvider() -// Encodes a point in time as a Unix epoch +// Timestamp encodes a point in time as a Unix epoch type Timestamp struct { - Time uint64 // Unix epoch timestamp, in seconds + Time uint64 `json:"time"` // Unix epoch timestamp, in seconds } // 8 bytes uint64 Time @@ -55,6 +56,18 @@ func (t *Timestamp) binaryPut(data []byte) error { return nil } +// UnmarshalJSON implements the json.Unmarshaller interface +func (t *Timestamp) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &t.Time) +} + +// MarshalJSON implements the json.Marshaller interface +func (t *Timestamp) MarshalJSON() ([]byte, error) { + return json.Marshal(t.Time) +} + +// DefaultTimestampProvider is a TimestampProvider that uses system time +// as time source type DefaultTimestampProvider struct { } diff --git a/swarm/storage/feed/topic.go b/swarm/storage/feed/topic.go new file mode 100644 index 0000000000..43a7b4ba4c --- /dev/null +++ b/swarm/storage/feed/topic.go @@ -0,0 +1,105 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// TopicLength establishes the max length of a topic string +const TopicLength = storage.AddressLength + +// Topic represents what a feed is about +type Topic [TopicLength]byte + +// ErrTopicTooLong is returned when creating a topic with a name/related content too long +var ErrTopicTooLong = fmt.Errorf("Topic is too long. Max length is %d", TopicLength) + +// NewTopic creates a new topic from a provided name and "related content" byte array, +// merging the two together. +// If relatedContent or name are longer than TopicLength, they will be truncated and an error returned +// name can be an empty string +// relatedContent can be nil +func NewTopic(name string, relatedContent []byte) (topic Topic, err error) { + if relatedContent != nil { + contentLength := len(relatedContent) + if contentLength > TopicLength { + contentLength = TopicLength + err = ErrTopicTooLong + } + copy(topic[:], relatedContent[:contentLength]) + } + nameBytes := []byte(name) + nameLength := len(nameBytes) + if nameLength > TopicLength { + nameLength = TopicLength + err = ErrTopicTooLong + } + bitutil.XORBytes(topic[:], topic[:], nameBytes[:nameLength]) + return topic, err +} + +// Hex will return the topic encoded as an hex string +func (t *Topic) Hex() string { + return hexutil.Encode(t[:]) +} + +// FromHex will parse a hex string into this Topic instance +func (t *Topic) FromHex(hex string) error { + bytes, err := hexutil.Decode(hex) + if err != nil || len(bytes) != len(t) { + return NewErrorf(ErrInvalidValue, "Cannot decode topic") + } + copy(t[:], bytes) + return nil +} + +// Name will try to extract the topic name out of the Topic +func (t *Topic) Name(relatedContent []byte) string { + nameBytes := *t + if relatedContent != nil { + contentLength := len(relatedContent) + if contentLength > TopicLength { + contentLength = TopicLength + } + bitutil.XORBytes(nameBytes[:], t[:], relatedContent[:contentLength]) + } + z := bytes.IndexByte(nameBytes[:], 0) + if z < 0 { + z = TopicLength + } + return string(nameBytes[:z]) + +} + +// UnmarshalJSON implements the json.Unmarshaller interface +func (t *Topic) UnmarshalJSON(data []byte) error { + var hex string + json.Unmarshal(data, &hex) + return t.FromHex(hex) +} + +// MarshalJSON implements the json.Marshaller interface +func (t *Topic) MarshalJSON() ([]byte, error) { + return json.Marshal(t.Hex()) +} diff --git a/swarm/storage/feed/topic_test.go b/swarm/storage/feed/topic_test.go new file mode 100644 index 0000000000..0403204f7f --- /dev/null +++ b/swarm/storage/feed/topic_test.go @@ -0,0 +1,50 @@ +package feed + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func TestTopic(t *testing.T) { + related, _ := hexutil.Decode("0xabcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789") + topicName := "test-topic" + topic, _ := NewTopic(topicName, related) + hex := topic.Hex() + expectedHex := "0xdfa89c750e3108f9c2aeef0123456789abcdef0123456789abcdef0123456789" + if hex != expectedHex { + t.Fatalf("Expected %s, got %s", expectedHex, hex) + } + + var topic2 Topic + topic2.FromHex(hex) + if topic2 != topic { + t.Fatal("Expected recovered topic to be equal to original one") + } + + if topic2.Name(related) != topicName { + t.Fatal("Retrieved name does not match") + } + + bytes, err := topic2.MarshalJSON() + if err != nil { + t.Fatal(err) + } + expectedJSON := `"0xdfa89c750e3108f9c2aeef0123456789abcdef0123456789abcdef0123456789"` + equal, err := areEqualJSON(expectedJSON, string(bytes)) + if err != nil { + t.Fatal(err) + } + if !equal { + t.Fatalf("Expected JSON to be %s, got %s", expectedJSON, string(bytes)) + } + + err = topic2.UnmarshalJSON(bytes) + if err != nil { + t.Fatal(err) + } + if topic2 != topic { + t.Fatal("Expected recovered topic to be equal to original one") + } + +} diff --git a/swarm/storage/feed/update.go b/swarm/storage/feed/update.go new file mode 100644 index 0000000000..21c004ca42 --- /dev/null +++ b/swarm/storage/feed/update.go @@ -0,0 +1,134 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/swarm/chunk" +) + +// ProtocolVersion defines the current version of the protocol that will be included in each update message +const ProtocolVersion uint8 = 0 + +const headerLength = 8 + +// Header defines a update message header including a protocol version byte +type Header struct { + Version uint8 // Protocol version + Padding [headerLength - 1]uint8 // reserved for future use +} + +// Update encapsulates the information sent as part of a feed update +type Update struct { + Header Header // + ID // Feed Update identifying information + data []byte // actual data payload +} + +const minimumUpdateDataLength = idLength + headerLength + 1 + +//MaxUpdateDataLength indicates the maximum payload size for a feed update +const MaxUpdateDataLength = chunk.DefaultSize - signatureLength - idLength - headerLength + +// binaryPut serializes the feed update information into the given slice +func (r *Update) binaryPut(serializedData []byte) error { + datalength := len(r.data) + if datalength == 0 { + return NewError(ErrInvalidValue, "a feed update must contain data") + } + + if datalength > MaxUpdateDataLength { + return NewErrorf(ErrInvalidValue, "feed update data is too big (length=%d). Max length=%d", datalength, MaxUpdateDataLength) + } + + if len(serializedData) != r.binaryLength() { + return NewErrorf(ErrInvalidValue, "slice passed to putBinary must be of exact size. Expected %d bytes", r.binaryLength()) + } + + var cursor int + // serialize Header + serializedData[cursor] = r.Header.Version + copy(serializedData[cursor+1:headerLength], r.Header.Padding[:headerLength-1]) + cursor += headerLength + + // serialize ID + if err := r.ID.binaryPut(serializedData[cursor : cursor+idLength]); err != nil { + return err + } + cursor += idLength + + // add the data + copy(serializedData[cursor:], r.data) + cursor += datalength + + return nil +} + +// binaryLength returns the expected number of bytes this structure will take to encode +func (r *Update) binaryLength() int { + return idLength + headerLength + len(r.data) +} + +// binaryGet populates this instance from the information contained in the passed byte slice +func (r *Update) binaryGet(serializedData []byte) error { + if len(serializedData) < minimumUpdateDataLength { + return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a feed update chunk", minimumUpdateDataLength) + } + dataLength := len(serializedData) - idLength - headerLength + // at this point we can be satisfied that we have the correct data length to read + + var cursor int + + // deserialize Header + r.Header.Version = serializedData[cursor] // extract the protocol version + copy(r.Header.Padding[:headerLength-1], serializedData[cursor+1:headerLength]) // extract the padding + cursor += headerLength + + if err := r.ID.binaryGet(serializedData[cursor : cursor+idLength]); err != nil { + return err + } + cursor += idLength + + data := serializedData[cursor : cursor+dataLength] + cursor += dataLength + + // now that all checks have passed, copy data into structure + r.data = make([]byte, dataLength) + copy(r.data, data) + + return nil + +} + +// FromValues deserializes this instance from a string key-value store +// useful to parse query strings +func (r *Update) FromValues(values Values, data []byte) error { + r.data = data + version, _ := strconv.ParseUint(values.Get("protocolVersion"), 10, 32) + r.Header.Version = uint8(version) + return r.ID.FromValues(values) +} + +// AppendValues serializes this structure into the provided string key-value store +// useful to build query strings +func (r *Update) AppendValues(values Values) []byte { + r.ID.AppendValues(values) + values.Set("protocolVersion", fmt.Sprintf("%d", r.Header.Version)) + return r.data +} diff --git a/swarm/storage/feed/update_test.go b/swarm/storage/feed/update_test.go new file mode 100644 index 0000000000..24c09b3617 --- /dev/null +++ b/swarm/storage/feed/update_test.go @@ -0,0 +1,50 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package feed + +import ( + "testing" +) + +func getTestFeedUpdate() *Update { + return &Update{ + ID: *getTestID(), + data: []byte("El que lee mucho y anda mucho, ve mucho y sabe mucho"), + } +} + +func TestUpdateSerializer(t *testing.T) { + testBinarySerializerRecovery(t, getTestFeedUpdate(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019456c20717565206c6565206d7563686f207920616e6461206d7563686f2c207665206d7563686f20792073616265206d7563686f") +} + +func TestUpdateLengthCheck(t *testing.T) { + testBinarySerializerLengthCheck(t, getTestFeedUpdate()) + // Test fail if update is too big + update := getTestFeedUpdate() + update.data = make([]byte, MaxUpdateDataLength+100) + serialized := make([]byte, update.binaryLength()) + if err := update.binaryPut(serialized); err == nil { + t.Fatal("Expected update.binaryPut to fail since update is too big") + } + + // test fail if data is empty or nil + update.data = nil + serialized = make([]byte, update.binaryLength()) + if err := update.binaryPut(serialized); err == nil { + t.Fatal("Expected update.binaryPut to fail since data is empty") + } +} diff --git a/swarm/storage/filestore_test.go b/swarm/storage/filestore_test.go index d79efb530d..fb0f761a4a 100644 --- a/swarm/storage/filestore_test.go +++ b/swarm/storage/filestore_test.go @@ -23,9 +23,11 @@ import ( "io/ioutil" "os" "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" ) -const testDataSize = 0x1000000 +const testDataSize = 0x0001000 func TestFileStorerandom(t *testing.T) { testFileStoreRandom(false, t) @@ -49,9 +51,9 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { fileStore := NewFileStore(localStore, NewFileStoreParams()) defer os.RemoveAll("/tmp/bzz") - reader, slice := GenerateRandomData(testDataSize) + slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() - key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) + key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) if err != nil { t.Fatalf("Store error: %v", err) } @@ -63,13 +65,13 @@ func testFileStoreRandom(toEncrypt bool, t *testing.T) { if isEncrypted != toEncrypt { t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) } - resultSlice := make([]byte, len(slice)) + resultSlice := make([]byte, testDataSize) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { t.Fatalf("Retrieve error: %v", err) } - if n != len(slice) { - t.Fatalf("Slice size error got %d, expected %d.", n, len(slice)) + if n != testDataSize { + t.Fatalf("Slice size error got %d, expected %d.", n, testDataSize) } if !bytes.Equal(slice, resultSlice) { t.Fatalf("Comparison error.") @@ -114,9 +116,9 @@ func testFileStoreCapacity(toEncrypt bool, t *testing.T) { DbStore: db, } fileStore := NewFileStore(localStore, NewFileStoreParams()) - reader, slice := GenerateRandomData(testDataSize) + slice := testutil.RandomBytes(1, testDataSize) ctx := context.TODO() - key, wait, err := fileStore.Store(ctx, reader, testDataSize, toEncrypt) + key, wait, err := fileStore.Store(ctx, bytes.NewReader(slice), testDataSize, toEncrypt) if err != nil { t.Errorf("Store error: %v", err) } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 879622b9a0..ff18e64c71 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -32,10 +32,13 @@ type hasherStore struct { hashFunc SwarmHasher hashSize int // content hash size refSize int64 // reference size (content hash + possibly encryption key) - nrChunks uint64 // number of chunks to store errC chan error // global error channel doneC chan struct{} // closed by Close() call to indicate that count is the final number of chunks quitC chan struct{} // closed to quit unterminated routines + // nrChunks is used with atomic functions + // it is required to be at the end of the struct to ensure 64bit alignment for arm architecture + // see: https://golang.org/pkg/sync/atomic/#pkg-note-BUG + nrChunks uint64 // number of chunks to store } // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index bde627394e..bd4f6b9162 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -32,21 +32,23 @@ import ( "fmt" "io" "io/ioutil" - "sort" "sync" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/opt" ) const ( - gcArrayFreeRatio = 0.1 - maxGCitems = 5000 // max number of items to be gc'd per call to collectGarbage() + defaultGCRatio = 10 + defaultMaxGCRound = 10000 + defaultMaxGCBatch = 5000 + + wEntryCnt = 1 << 0 + wIndexCnt = 1 << 1 + wAccessCnt = 1 << 2 ) var ( @@ -55,25 +57,19 @@ var ( var ( keyIndex = byte(0) - keyOldData = byte(1) keyAccessCnt = []byte{2} keyEntryCnt = []byte{3} keyDataIdx = []byte{4} keyData = byte(6) keyDistanceCnt = byte(7) + keySchema = []byte{8} + keyGCIdx = byte(9) // access to chunk data index, used by garbage collection in ascending order from first entry ) var ( ErrDBClosed = errors.New("LDBStore closed") ) -type gcItem struct { - idx uint64 - value uint64 - idxKey []byte - po uint8 -} - type LDBStoreParams struct { *StoreParams Path string @@ -89,6 +85,16 @@ func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams { } } +type garbage struct { + maxRound int // maximum number of chunks to delete in one garbage collection round + maxBatch int // maximum number of chunks to delete in one db request batch + ratio int // 1/x ratio to calculate the number of chunks to gc on a low capacity db + count int // number of chunks deleted in running round + target int // number of chunks to delete in running round + batch *dbBatch // the delete batch + runC chan struct{} // struct in chan means gc is NOT running +} + type LDBStore struct { db *LDBDatabase @@ -102,12 +108,12 @@ type LDBStore struct { hashfunc SwarmHasher po func(Address) uint8 - batchC chan bool batchesC chan struct{} closed bool batch *dbBatch lock sync.RWMutex quit chan struct{} + gc *garbage // Functions encodeDataFunc is used to bypass // the default functionality of DbStore with @@ -166,9 +172,47 @@ func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { data, _ = s.db.Get(keyDataIdx) s.dataIdx = BytesToU64(data) + // set up garbage collection + s.gc = &garbage{ + maxBatch: defaultMaxGCBatch, + maxRound: defaultMaxGCRound, + ratio: defaultGCRatio, + } + + s.gc.runC = make(chan struct{}, 1) + s.gc.runC <- struct{}{} + return s, nil } +// MarkAccessed increments the access counter as a best effort for a chunk, so +// the chunk won't get garbage collected. +func (s *LDBStore) MarkAccessed(addr Address) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.closed { + return + } + + proximity := s.po(addr) + s.tryAccessIdx(addr, proximity) +} + +// initialize and set values for processing of gc round +func (s *LDBStore) startGC(c int) { + + s.gc.count = 0 + // calculate the target number of deletions + if c >= s.gc.maxRound { + s.gc.target = s.gc.maxRound + } else { + s.gc.target = c / s.gc.ratio + } + s.gc.batch = newBatch() + log.Debug("startgc", "requested", c, "target", s.gc.target) +} + // NewMockDbStore creates a new instance of DbStore with // mockStore set to a provided value. If mockStore argument is nil, // this function behaves exactly as NewDbStore. @@ -225,6 +269,35 @@ func getDataKey(idx uint64, po uint8) []byte { return key } +func getGCIdxKey(index *dpaDBIndex) []byte { + key := make([]byte, 9) + key[0] = keyGCIdx + binary.BigEndian.PutUint64(key[1:], index.Access) + return key +} + +func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { + val := make([]byte, 41) // po = 1, index.Index = 8, Address = 32 + val[0] = po + binary.BigEndian.PutUint64(val[1:], index.Idx) + copy(val[9:], addr) + return val +} + +func parseIdxKey(key []byte) (byte, []byte) { + return key[0], key[1:] +} + +func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) { + index = &dpaDBIndex{ + Idx: binary.BigEndian.Uint64(val[1:]), + Access: binary.BigEndian.Uint64(accessCnt), + } + po = val[0] + addr = val[9:] + return +} + func encodeIndex(index *dpaDBIndex) []byte { data, _ := rlp.EncodeToBytes(index) return data @@ -247,55 +320,71 @@ func decodeData(addr Address, data []byte) (*chunk, error) { return NewChunk(addr, data[32:]), nil } -func (s *LDBStore) collectGarbage(ratio float32) { - log.Trace("collectGarbage", "ratio", ratio) +func (s *LDBStore) collectGarbage() error { + + // prevent duplicate gc from starting when one is already running + select { + case <-s.gc.runC: + default: + return nil + } + + s.lock.Lock() + entryCnt := s.entryCnt + s.lock.Unlock() metrics.GetOrRegisterCounter("ldbstore.collectgarbage", nil).Inc(1) - it := s.db.NewIterator() - defer it.Release() + // calculate the amount of chunks to collect and reset counter + s.startGC(int(entryCnt)) + log.Debug("collectGarbage", "target", s.gc.target, "entryCnt", entryCnt) - garbage := []*gcItem{} - gcnt := 0 + var totalDeleted int + for s.gc.count < s.gc.target { + it := s.db.NewIterator() + ok := it.Seek([]byte{keyGCIdx}) + var singleIterationCount int - for ok := it.Seek([]byte{keyIndex}); ok && (gcnt < maxGCitems) && (uint64(gcnt) < s.entryCnt); ok = it.Next() { - itkey := it.Key() + // every batch needs a lock so we avoid entries changing accessidx in the meantime + s.lock.Lock() + for ; ok && (singleIterationCount < s.gc.maxBatch); ok = it.Next() { - if (itkey == nil) || (itkey[0] != keyIndex) { - break + // quit if no more access index keys + itkey := it.Key() + if (itkey == nil) || (itkey[0] != keyGCIdx) { + break + } + + // get chunk data entry from access index + val := it.Value() + index, po, hash := parseGCIdxEntry(itkey[1:], val) + keyIdx := make([]byte, 33) + keyIdx[0] = keyIndex + copy(keyIdx[1:], hash) + + // add delete operation to batch + s.delete(s.gc.batch.Batch, index, keyIdx, po) + singleIterationCount++ + s.gc.count++ + log.Trace("garbage collect enqueued chunk for deletion", "key", hash) + + // break if target is not on max garbage batch boundary + if s.gc.count >= s.gc.target { + break + } } - // it.Key() contents change on next call to it.Next(), so we must copy it - key := make([]byte, len(it.Key())) - copy(key, it.Key()) - - val := it.Value() - - var index dpaDBIndex - - hash := key[1:] - decodeIndex(val, &index) - po := s.po(hash) - - gci := &gcItem{ - idxKey: key, - idx: index.Idx, - value: index.Access, // the smaller, the more likely to be gc'd. see sort comparator below. - po: po, - } - - garbage = append(garbage, gci) - gcnt++ + s.writeBatch(s.gc.batch, wEntryCnt) + s.lock.Unlock() + it.Release() + log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count) } - sort.Slice(garbage[:gcnt], func(i, j int) bool { return garbage[i].value < garbage[j].value }) + s.gc.runC <- struct{}{} + log.Debug("garbage collect done", "c", s.gc.count) - cutoff := int(float32(gcnt) * ratio) - metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(cutoff)) - - for i := 0; i < cutoff; i++ { - s.delete(garbage[i].idx, garbage[i].idxKey, garbage[i].po) - } + metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(totalDeleted)) + return nil } // Export writes all chunks from the store to a tar archive, returning the @@ -418,8 +507,8 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } } -func (s *LDBStore) Cleanup() { - //Iterates over the database and checks that there are no chunks bigger than 4kb +// Cleanup iterates over the database and deletes chunks if they pass the `f` condition +func (s *LDBStore) Cleanup(f func(*chunk) bool) { var errorsFound, removed, total int it := s.db.NewIterator() @@ -471,9 +560,10 @@ func (s *LDBStore) Cleanup() { cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) - if len(c.sdata) > ch.DefaultSize+8 { + // if chunk is to be removed + if f(c) { log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) - s.delete(index.Idx, getIndexKey(key[1:]), po) + s.deleteNow(&index, getIndexKey(key[1:]), po) removed++ errorsFound++ } @@ -482,67 +572,196 @@ func (s *LDBStore) Cleanup() { log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) } -func (s *LDBStore) ReIndex() { - //Iterates over the database and checks that there are no faulty chunks +// CleanGCIndex rebuilds the garbage collector index from scratch, while +// removing inconsistent elements, e.g., indices with missing data chunks. +// WARN: it's a pretty heavy, long running function. +func (s *LDBStore) CleanGCIndex() error { + s.lock.Lock() + defer s.lock.Unlock() + + batch := leveldb.Batch{} + + var okEntryCount uint64 + var totalEntryCount uint64 + + // throw out all gc indices, we will rebuild from cleaned index it := s.db.NewIterator() - startPosition := []byte{keyOldData} - it.Seek(startPosition) - var key []byte - var errorsFound, total int + it.Seek([]byte{keyGCIdx}) + var gcDeletes int for it.Valid() { - key = it.Key() - if (key == nil) || (key[0] != keyOldData) { + rowType, _ := parseIdxKey(it.Key()) + if rowType != keyGCIdx { break } - data := it.Value() - hasher := s.hashfunc() - hasher.Write(data) - hash := hasher.Sum(nil) - - newKey := make([]byte, 10) - oldCntKey := make([]byte, 2) - newCntKey := make([]byte, 2) - oldCntKey[0] = keyDistanceCnt - newCntKey[0] = keyDistanceCnt - key[0] = keyData - key[1] = s.po(Address(key[1:])) - oldCntKey[1] = key[1] - newCntKey[1] = s.po(Address(newKey[1:])) - copy(newKey[2:], key[1:]) - newValue := append(hash, data...) - - batch := new(leveldb.Batch) - batch.Delete(key) - s.bucketCnt[oldCntKey[1]]-- - batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]])) - batch.Put(newKey, newValue) - s.bucketCnt[newCntKey[1]]++ - batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]])) - s.db.Write(batch) + batch.Delete(it.Key()) + gcDeletes++ it.Next() } + log.Debug("gc", "deletes", gcDeletes) + if err := s.db.Write(&batch); err != nil { + return err + } + batch.Reset() + it.Release() - log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) + + // corrected po index pointer values + var poPtrs [256]uint64 + + // set to true if chunk count not on 4096 iteration boundary + var doneIterating bool + + // last key index in previous iteration + lastIdxKey := []byte{keyIndex} + + // counter for debug output + var cleanBatchCount int + + // go through all key index entries + for !doneIterating { + cleanBatchCount++ + var idxs []dpaDBIndex + var chunkHashes [][]byte + var pos []uint8 + it := s.db.NewIterator() + + it.Seek(lastIdxKey) + + // 4096 is just a nice number, don't look for any hidden meaning here... + var i int + for i = 0; i < 4096; i++ { + + // this really shouldn't happen unless database is empty + // but let's keep it to be safe + if !it.Valid() { + doneIterating = true + break + } + + // if it's not keyindex anymore we're done iterating + rowType, chunkHash := parseIdxKey(it.Key()) + if rowType != keyIndex { + doneIterating = true + break + } + + // decode the retrieved index + var idx dpaDBIndex + err := decodeIndex(it.Value(), &idx) + if err != nil { + return fmt.Errorf("corrupt index: %v", err) + } + po := s.po(chunkHash) + lastIdxKey = it.Key() + + // if we don't find the data key, remove the entry + // if we find it, add to the array of new gc indices to create + dataKey := getDataKey(idx.Idx, po) + _, err = s.db.Get(dataKey) + if err != nil { + log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) + batch.Delete(it.Key()) + } else { + idxs = append(idxs, idx) + chunkHashes = append(chunkHashes, chunkHash) + pos = append(pos, po) + okEntryCount++ + if idx.Idx > poPtrs[po] { + poPtrs[po] = idx.Idx + } + } + totalEntryCount++ + it.Next() + } + it.Release() + + // flush the key index corrections + err := s.db.Write(&batch) + if err != nil { + return err + } + batch.Reset() + + // add correct gc indices + for i, okIdx := range idxs { + gcIdxKey := getGCIdxKey(&okIdx) + gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i]) + batch.Put(gcIdxKey, gcIdxData) + log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData) + } + + // flush them + err = s.db.Write(&batch) + if err != nil { + return err + } + batch.Reset() + + log.Debug("clean gc index pass", "batch", cleanBatchCount, "checked", i, "kept", len(idxs)) + } + + log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len()) + + // lastly add updated entry count + var entryCount [8]byte + binary.BigEndian.PutUint64(entryCount[:], okEntryCount) + batch.Put(keyEntryCnt, entryCount[:]) + + // and add the new po index pointers + var poKey [2]byte + poKey[0] = keyDistanceCnt + for i, poPtr := range poPtrs { + poKey[1] = uint8(i) + if poPtr == 0 { + batch.Delete(poKey[:]) + } else { + var idxCount [8]byte + binary.BigEndian.PutUint64(idxCount[:], poPtr) + batch.Put(poKey[:], idxCount[:]) + } + } + + // if you made it this far your harddisk has survived. Congratulations + return s.db.Write(&batch) } -func (s *LDBStore) Delete(addr Address) { +// Delete is removes a chunk and updates indices. +// Is thread safe +func (s *LDBStore) Delete(addr Address) error { s.lock.Lock() defer s.lock.Unlock() ikey := getIndexKey(addr) - var indx dpaDBIndex - s.tryAccessIdx(ikey, &indx) + idata, err := s.db.Get(ikey) + if err != nil { + return err + } - s.delete(indx.Idx, ikey, s.po(addr)) + var idx dpaDBIndex + decodeIndex(idata, &idx) + proximity := s.po(addr) + return s.deleteNow(&idx, ikey, proximity) } -func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { +// executes one delete operation immediately +// see *LDBStore.delete +func (s *LDBStore) deleteNow(idx *dpaDBIndex, idxKey []byte, po uint8) error { + batch := new(leveldb.Batch) + s.delete(batch, idx, idxKey, po) + return s.db.Write(batch) +} + +// adds a delete chunk operation to the provided batch +// if called directly, decrements entrycount regardless if the chunk exists upon deletion. Risk of wrap to max uint64 +func (s *LDBStore) delete(batch *leveldb.Batch, idx *dpaDBIndex, idxKey []byte, po uint8) { metrics.GetOrRegisterCounter("ldbstore.delete", nil).Inc(1) - batch := new(leveldb.Batch) + gcIdxKey := getGCIdxKey(idx) + batch.Delete(gcIdxKey) + dataKey := getDataKey(idx.Idx, po) + batch.Delete(dataKey) batch.Delete(idxKey) - batch.Delete(getDataKey(idx, po)) s.entryCnt-- dbEntryCount.Dec(1) cntKey := make([]byte, 2) @@ -550,7 +769,6 @@ func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { cntKey[1] = po batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) - s.db.Write(batch) } func (s *LDBStore) BinIndex(po uint8) uint64 { @@ -571,6 +789,9 @@ func (s *LDBStore) CurrentStorageIndex() uint64 { return s.dataIdx } +// Put adds a chunk to the database, adding indices and incrementing global counters. +// If it already exists, it merely increments the access count of the existing entry. +// Is thread safe func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { metrics.GetOrRegisterCounter("ldbstore.put", nil).Inc(1) log.Trace("ldbstore.put", "key", chunk.Address()) @@ -592,15 +813,14 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { idata, err := s.db.Get(ikey) if err != nil { s.doPut(chunk, &index, po) - } else { - log.Trace("ldbstore.put: chunk already exists, only update access", "key", chunk.Address) - decodeIndex(idata, &index) } - index.Access = s.accessCnt - s.accessCnt++ idata = encodeIndex(&index) s.batch.Put(ikey, idata) + // add the access-chunkindex index for garbage collection + gcIdxKey := getGCIdxKey(&index) + gcIdxData := getGCIdxValue(&index, po, chunk.Address()) + s.batch.Put(gcIdxKey, gcIdxData) s.lock.Unlock() select { @@ -616,7 +836,7 @@ func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { } } -// force putting into db, does not check access index +// force putting into db, does not check or update necessary indices func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { data := s.encodeDataFunc(chunk) dkey := getDataKey(s.dataIdx, po) @@ -626,7 +846,8 @@ func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { s.entryCnt++ dbEntryCount.Inc(1) s.dataIdx++ - + index.Access = s.accessCnt + s.accessCnt++ cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po @@ -658,38 +879,26 @@ func (s *LDBStore) writeCurrentBatch() error { if l == 0 { return nil } - e := s.entryCnt - d := s.dataIdx - a := s.accessCnt s.batch = newBatch() - b.err = s.writeBatch(b, e, d, a) + b.err = s.writeBatch(b, wEntryCnt|wAccessCnt|wIndexCnt) close(b.c) - for e > s.capacity { - log.Trace("for >", "e", e, "s.capacity", s.capacity) - // Collect garbage in a separate goroutine - // to be able to interrupt this loop by s.quit. - done := make(chan struct{}) - go func() { - s.collectGarbage(gcArrayFreeRatio) - log.Trace("collectGarbage closing done") - close(done) - }() - - select { - case <-s.quit: - return errors.New("CollectGarbage terminated due to quit") - case <-done: - } - e = s.entryCnt + if s.entryCnt >= s.capacity { + go s.collectGarbage() } return nil } // must be called non concurrently -func (s *LDBStore) writeBatch(b *dbBatch, entryCnt, dataIdx, accessCnt uint64) error { - b.Put(keyEntryCnt, U64ToBytes(entryCnt)) - b.Put(keyDataIdx, U64ToBytes(dataIdx)) - b.Put(keyAccessCnt, U64ToBytes(accessCnt)) +func (s *LDBStore) writeBatch(b *dbBatch, wFlag uint8) error { + if wFlag&wEntryCnt > 0 { + b.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) + } + if wFlag&wIndexCnt > 0 { + b.Put(keyDataIdx, U64ToBytes(s.dataIdx)) + } + if wFlag&wAccessCnt > 0 { + b.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) + } l := b.Len() if err := s.db.Write(b.Batch); err != nil { return fmt.Errorf("unable to write batch: %v", err) @@ -711,25 +920,62 @@ func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { } } -// try to find index; if found, update access cnt and return true -func (s *LDBStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { +// tryAccessIdx tries to find index entry. If found then increments the access +// count for garbage collection and returns the index entry and true for found, +// otherwise returns nil and false. +func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) { + ikey := getIndexKey(addr) idata, err := s.db.Get(ikey) if err != nil { - return false + return nil, false } + + index := new(dpaDBIndex) decodeIndex(idata, index) + oldGCIdxKey := getGCIdxKey(index) s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) - s.accessCnt++ index.Access = s.accessCnt idata = encodeIndex(index) + s.accessCnt++ s.batch.Put(ikey, idata) + newGCIdxKey := getGCIdxKey(index) + newGCIdxData := getGCIdxValue(index, po, ikey[1:]) + s.batch.Delete(oldGCIdxKey) + s.batch.Put(newGCIdxKey, newGCIdxData) select { case s.batchesC <- struct{}{}: default: } - return true + return index, true } +// GetSchema is returning the current named schema of the datastore as read from LevelDB +func (s *LDBStore) GetSchema() (string, error) { + s.lock.Lock() + defer s.lock.Unlock() + + data, err := s.db.Get(keySchema) + if err != nil { + if err == leveldb.ErrNotFound { + return DbSchemaNone, nil + } + return "", err + } + + return string(data), nil +} + +// PutSchema is saving a named schema to the LevelDB datastore +func (s *LDBStore) PutSchema(schema string) error { + s.lock.Lock() + defer s.lock.Unlock() + + return s.db.Put(keySchema, []byte(schema)) +} + +// Get retrieves the chunk matching the provided key from the database. +// If the chunk entry does not exist, it returns an error +// Updates access count and is thread safe func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) { metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1) log.Trace("ldbstore.get", "key", addr) @@ -739,12 +985,14 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) return s.get(addr) } +// TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { - var indx dpaDBIndex if s.closed { return nil, ErrDBClosed } - if s.tryAccessIdx(getIndexKey(addr), &indx) { + proximity := s.po(addr) + index, found := s.tryAccessIdx(addr, proximity) + if found { var data []byte if s.getDataFunc != nil { // if getDataFunc is defined, use it to retrieve the chunk data @@ -755,13 +1003,12 @@ func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { } } else { // default DbStore functionality to retrieve chunk data - proximity := s.po(addr) - datakey := getDataKey(indx.Idx, proximity) + datakey := getDataKey(index.Idx, proximity) data, err = s.db.Get(datakey) - log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", indx.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) + log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", index.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) if err != nil { log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err) - s.delete(indx.Idx, getIndexKey(addr), s.po(addr)) + s.deleteNow(index, getIndexKey(addr), s.po(addr)) return } } @@ -788,33 +1035,14 @@ func newMockGetDataFunc(mockStore *mock.NodeStore) func(addr Address) (data []by } } -func (s *LDBStore) updateAccessCnt(addr Address) { - - s.lock.Lock() - defer s.lock.Unlock() - - var index dpaDBIndex - s.tryAccessIdx(getIndexKey(addr), &index) // result_chn == nil, only update access cnt - -} - func (s *LDBStore) setCapacity(c uint64) { s.lock.Lock() defer s.lock.Unlock() s.capacity = c - if s.entryCnt > c { - ratio := float32(1.01) - float32(c)/float32(s.entryCnt) - if ratio < gcArrayFreeRatio { - ratio = gcArrayFreeRatio - } - if ratio > 1 { - ratio = 1 - } - for s.entryCnt > c { - s.collectGarbage(ratio) - } + for s.entryCnt > c { + s.collectGarbage() } } @@ -854,15 +1082,3 @@ func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Add } return it.Error() } - -func databaseExists(path string) bool { - o := &opt.Options{ - ErrorIfMissing: true, - } - tdb, err := leveldb.OpenFile(path, o) - if err != nil { - return false - } - defer tdb.Close() - return true -} diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 75b5d6aa95..e8b9ae39bc 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -19,9 +19,12 @@ package storage import ( "bytes" "context" + "encoding/binary" "fmt" "io/ioutil" "os" + "strconv" + "strings" "testing" "time" @@ -29,7 +32,6 @@ import ( ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" - ldberrors "github.com/syndtr/goleveldb/leveldb/errors" ) @@ -103,6 +105,46 @@ func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) { testStoreCorrect(db, n, chunksize, t) } +func TestMarkAccessed(t *testing.T) { + db, cleanup, err := newTestDbStore(false, true) + defer cleanup() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + + h := GenerateRandomChunk(ch.DefaultSize) + + db.Put(context.Background(), h) + + var index dpaDBIndex + addr := h.Address() + idxk := getIndexKey(addr) + + idata, err := db.db.Get(idxk) + if err != nil { + t.Fatal(err) + } + decodeIndex(idata, &index) + + if index.Access != 0 { + t.Fatalf("Expected the access index to be %d, but it is %d", 0, index.Access) + } + + db.MarkAccessed(addr) + db.writeCurrentBatch() + + idata, err = db.db.Get(idxk) + if err != nil { + t.Fatal(err) + } + decodeIndex(idata, &index) + + if index.Access != 1 { + t.Fatalf("Expected the access index to be %d, but it is %d", 1, index.Access) + } + +} + func TestDbStoreRandom_1(t *testing.T) { testDbStoreRandom(1, 0, false, t) } @@ -111,12 +153,12 @@ func TestDbStoreCorrect_1(t *testing.T) { testDbStoreCorrect(1, 4096, false, t) } -func TestDbStoreRandom_5k(t *testing.T) { - testDbStoreRandom(5000, 0, false, t) +func TestDbStoreRandom_1k(t *testing.T) { + testDbStoreRandom(1000, 0, false, t) } -func TestDbStoreCorrect_5k(t *testing.T) { - testDbStoreCorrect(5000, 4096, false, t) +func TestDbStoreCorrect_1k(t *testing.T) { + testDbStoreCorrect(1000, 4096, false, t) } func TestMockDbStoreRandom_1(t *testing.T) { @@ -127,12 +169,12 @@ func TestMockDbStoreCorrect_1(t *testing.T) { testDbStoreCorrect(1, 4096, true, t) } -func TestMockDbStoreRandom_5k(t *testing.T) { - testDbStoreRandom(5000, 0, true, t) +func TestMockDbStoreRandom_1k(t *testing.T) { + testDbStoreRandom(1000, 0, true, t) } -func TestMockDbStoreCorrect_5k(t *testing.T) { - testDbStoreCorrect(5000, 4096, true, t) +func TestMockDbStoreCorrect_1k(t *testing.T) { + testDbStoreCorrect(1000, 4096, true, t) } func testDbStoreNotFound(t *testing.T, mock bool) { @@ -297,27 +339,74 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { } // TestLDBStoreCollectGarbage tests that we can put more chunks than LevelDB's capacity, and -// retrieve only some of them, because garbage collection must have cleared some of them +// retrieve only some of them, because garbage collection must have partially cleared the store +// Also tests that we can delete chunks and that we can trigger garbage collection func TestLDBStoreCollectGarbage(t *testing.T) { - capacity := 500 - n := 2000 + + // below max ronud + initialCap := defaultMaxGCRound / 100 + cap := initialCap / 2 + t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) + t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) + + // at max round + cap = initialCap + t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) + t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) + + // more than max around, not on threshold + cap = initialCap + 500 + t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) + t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) + +} + +func testLDBStoreCollectGarbage(t *testing.T) { + params := strings.Split(t.Name(), "/") + capacity, err := strconv.Atoi(params[2]) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(params[3]) + if err != nil { + t.Fatal(err) + } ldb, cleanup := newLDBStore(t) ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) - if err != nil { - t.Fatal(err.Error()) + // retrieve the gc round target count for the db capacity + ldb.startGC(capacity) + roundTarget := ldb.gc.target + + // split put counts to gc target count threshold, and wait for gc to finish in between + var allChunks []Chunk + remaining := n + for remaining > 0 { + var putCount int + if remaining < roundTarget { + putCount = remaining + } else { + putCount = roundTarget + } + remaining -= putCount + chunks, err := mputRandomChunks(ldb, putCount, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) + } + allChunks = append(allChunks, chunks...) + log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) } - log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) - - // wait for garbage collection to kick in on the responsible actor - time.Sleep(1 * time.Second) + // attempt gets on all put chunks var missing int - for _, ch := range chunks { - ret, err := ldb.Get(context.Background(), ch.Address()) + for _, ch := range allChunks { + ret, err := ldb.Get(context.TODO(), ch.Address()) if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { missing++ continue @@ -333,8 +422,10 @@ func TestLDBStoreCollectGarbage(t *testing.T) { log.Trace("got back chunk", "chunk", ret) } - if missing < n-capacity { - t.Fatalf("gc failure: expected to miss %v chunks, but only %v are actually missing", n-capacity, missing) + // all surplus chunks should be missing + expectMissing := roundTarget + (((n - capacity) / roundTarget) * roundTarget) + if missing != expectMissing { + t.Fatalf("gc failure: expected to miss %v chunks, but only %v are actually missing", expectMissing, missing) } log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) @@ -367,7 +458,6 @@ func TestLDBStoreAddRemove(t *testing.T) { if i%2 == 0 { // expect even chunks to be missing if err == nil { - // if err != ErrChunkNotFound { t.Fatal("expected chunk to be missing, but got no error") } } else { @@ -383,30 +473,48 @@ func TestLDBStoreAddRemove(t *testing.T) { } } -// TestLDBStoreRemoveThenCollectGarbage tests that we can delete chunks and that we can trigger garbage collection -func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { - capacity := 11 - surplus := 4 +func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { + + params := strings.Split(t.Name(), "/") + capacity, err := strconv.Atoi(params[2]) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(params[3]) + if err != nil { + t.Fatal(err) + } ldb, cleanup := newLDBStore(t) + defer cleanup() ldb.setCapacity(uint64(capacity)) - n := capacity - - chunks := []Chunk{} - for i := 0; i < n+surplus; i++ { + // put capacity count number of chunks + chunks := make([]Chunk, n) + for i := 0; i < n; i++ { c := GenerateRandomChunk(ch.DefaultSize) - chunks = append(chunks, c) + chunks[i] = c log.Trace("generate random chunk", "idx", i, "chunk", c) } for i := 0; i < n; i++ { - ldb.Put(context.TODO(), chunks[i]) + err := ldb.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatal(err) + } } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) + // delete all chunks + // (only count the ones actually deleted, the rest will have been gc'd) + deletes := 0 for i := 0; i < n; i++ { - ldb.Delete(chunks[i].Address()) + if ldb.Delete(chunks[i].Address()) == nil { + deletes++ + } } log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) @@ -415,37 +523,49 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { t.Fatalf("ldb.entrCnt expected 0 got %v", ldb.entryCnt) } - expAccessCnt := uint64(n * 2) + // the manual deletes will have increased accesscnt, so we need to add this when we verify the current count + expAccessCnt := uint64(n) if ldb.accessCnt != expAccessCnt { - t.Fatalf("ldb.accessCnt expected %v got %v", expAccessCnt, ldb.entryCnt) + t.Fatalf("ldb.accessCnt expected %v got %v", expAccessCnt, ldb.accessCnt) } - cleanup() + // retrieve the gc round target count for the db capacity + ldb.startGC(capacity) + roundTarget := ldb.gc.target - ldb, cleanup = newLDBStore(t) - capacity = 10 - ldb.setCapacity(uint64(capacity)) - defer cleanup() + remaining := n + var puts int + for remaining > 0 { + var putCount int + if remaining < roundTarget { + putCount = remaining + } else { + putCount = roundTarget + } + remaining -= putCount + for putCount > 0 { + ldb.Put(context.TODO(), chunks[puts]) + log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n, "puts", puts, "remaining", remaining, "roundtarget", roundTarget) + puts++ + putCount-- + } - n = capacity + surplus - - for i := 0; i < n; i++ { - ldb.Put(context.TODO(), chunks[i]) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) } - // wait for garbage collection - time.Sleep(1 * time.Second) - // expect first surplus chunks to be missing, because they have the smallest access value - for i := 0; i < surplus; i++ { + expectMissing := roundTarget + (((n - capacity) / roundTarget) * roundTarget) + for i := 0; i < expectMissing; i++ { _, err := ldb.Get(context.TODO(), chunks[i].Address()) if err == nil { - t.Fatal("expected surplus chunk to be missing, but got no error") + t.Fatalf("expected surplus chunk %d to be missing, but got no error", i) } } // expect last chunks to be present, as they have the largest access value - for i := surplus; i < surplus+capacity; i++ { + for i := expectMissing; i < n; i++ { ret, err := ldb.Get(context.TODO(), chunks[i].Address()) if err != nil { t.Fatalf("chunk %v: expected no error, but got %s", i, err) @@ -455,3 +575,228 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { } } } + +// TestLDBStoreCollectGarbageAccessUnlikeIndex tests garbage collection where accesscount differs from indexcount +func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { + + capacity := defaultMaxGCRound / 100 * 2 + n := capacity - 1 + + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(uint64(capacity)) + defer cleanup() + + chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) + } + log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) + + // set first added capacity/2 chunks to highest accesscount + for i := 0; i < capacity/2; i++ { + _, err := ldb.Get(context.TODO(), chunks[i].Address()) + if err != nil { + t.Fatalf("fail add chunk #%d - %s: %v", i, chunks[i].Address(), err) + } + } + _, err = mputRandomChunks(ldb, 2, int64(ch.DefaultSize)) + if err != nil { + t.Fatal(err.Error()) + } + + // wait for garbage collection to kick in on the responsible actor + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + waitGc(ctx, ldb) + + var missing int + for i, ch := range chunks[2 : capacity/2] { + ret, err := ldb.Get(context.TODO(), ch.Address()) + if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { + t.Fatalf("fail find chunk #%d - %s: %v", i, ch.Address(), err) + } + + if !bytes.Equal(ret.Data(), ch.Data()) { + t.Fatal("expected to get the same data back, but got smth else") + } + log.Trace("got back chunk", "chunk", ret) + } + + log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) +} + +func TestCleanIndex(t *testing.T) { + capacity := 5000 + n := 3 + + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(uint64(capacity)) + defer cleanup() + + chunks, err := mputRandomChunks(ldb, n, 4096) + if err != nil { + t.Fatal(err) + } + + // remove the data of the first chunk + po := ldb.po(chunks[0].Address()[:]) + dataKey := make([]byte, 10) + dataKey[0] = keyData + dataKey[1] = byte(po) + // dataKey[2:10] = first chunk has storageIdx 0 on [2:10] + if _, err := ldb.db.Get(dataKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Delete(dataKey); err != nil { + t.Fatal(err) + } + + // remove the gc index row for the first chunk + gcFirstCorrectKey := make([]byte, 9) + gcFirstCorrectKey[0] = keyGCIdx + if err := ldb.db.Delete(gcFirstCorrectKey); err != nil { + t.Fatal(err) + } + + // warp the gc data of the second chunk + // this data should be correct again after the clean + gcSecondCorrectKey := make([]byte, 9) + gcSecondCorrectKey[0] = keyGCIdx + binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(1)) + gcSecondCorrectVal, err := ldb.db.Get(gcSecondCorrectKey) + if err != nil { + t.Fatal(err) + } + warpedGCVal := make([]byte, len(gcSecondCorrectVal)+1) + copy(warpedGCVal[1:], gcSecondCorrectVal) + if err := ldb.db.Delete(gcSecondCorrectKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Put(gcSecondCorrectKey, warpedGCVal); err != nil { + t.Fatal(err) + } + + if err := ldb.CleanGCIndex(); err != nil { + t.Fatal(err) + } + + // the index without corresponding data should have been deleted + idxKey := make([]byte, 33) + idxKey[0] = keyIndex + copy(idxKey[1:], chunks[0].Address()) + if _, err := ldb.db.Get(idxKey); err == nil { + t.Fatalf("expected chunk 0 idx to be pruned: %v", idxKey) + } + + // the two other indices should be present + copy(idxKey[1:], chunks[1].Address()) + if _, err := ldb.db.Get(idxKey); err != nil { + t.Fatalf("expected chunk 1 idx to be present: %v", idxKey) + } + + copy(idxKey[1:], chunks[2].Address()) + if _, err := ldb.db.Get(idxKey); err != nil { + t.Fatalf("expected chunk 2 idx to be present: %v", idxKey) + } + + // first gc index should still be gone + if _, err := ldb.db.Get(gcFirstCorrectKey); err == nil { + t.Fatalf("expected gc 0 idx to be pruned: %v", idxKey) + } + + // second gc index should still be fixed + if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { + t.Fatalf("expected gc 1 idx to be present: %v", idxKey) + } + + // third gc index should be unchanged + binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(2)) + if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + + c, err := ldb.db.Get(keyEntryCnt) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + + // entrycount should now be one less + entryCount := binary.BigEndian.Uint64(c) + if entryCount != 2 { + t.Fatalf("expected entrycnt to be 2, was %d", c) + } + + // the chunks might accidentally be in the same bin + // if so that bin counter will now be 2 - the highest added index. + // if not, the total of them will be 3 + poBins := []uint8{ldb.po(chunks[1].Address()), ldb.po(chunks[2].Address())} + if poBins[0] == poBins[1] { + poBins = poBins[:1] + } + + var binTotal uint64 + var currentBin [2]byte + currentBin[0] = keyDistanceCnt + if len(poBins) == 1 { + currentBin[1] = poBins[0] + c, err := ldb.db.Get(currentBin[:]) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + binCount := binary.BigEndian.Uint64(c) + if binCount != 2 { + t.Fatalf("expected entrycnt to be 2, was %d", binCount) + } + } else { + for _, bin := range poBins { + currentBin[1] = bin + c, err := ldb.db.Get(currentBin[:]) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + binCount := binary.BigEndian.Uint64(c) + binTotal += binCount + + } + if binTotal != 3 { + t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal) + } + } + + // check that the iterator quits properly + chunks, err = mputRandomChunks(ldb, 4100, 4096) + if err != nil { + t.Fatal(err) + } + + po = ldb.po(chunks[4099].Address()[:]) + dataKey = make([]byte, 10) + dataKey[0] = keyData + dataKey[1] = byte(po) + binary.BigEndian.PutUint64(dataKey[2:], 4099+3) + if _, err := ldb.db.Get(dataKey); err != nil { + t.Fatal(err) + } + if err := ldb.db.Delete(dataKey); err != nil { + t.Fatal(err) + } + + if err := ldb.CleanGCIndex(); err != nil { + t.Fatal(err) + } + + // entrycount should now be one less of added chunks + c, err = ldb.db.Get(keyEntryCnt) + if err != nil { + t.Fatalf("expected gc 2 idx to be present: %v", idxKey) + } + entryCount = binary.BigEndian.Uint64(c) + if entryCount != 4099+2 { + t.Fatalf("expected entrycnt to be 2, was %d", c) + } +} + +func waitGc(ctx context.Context, ldb *LDBStore) { + <-ldb.gc.runC + ldb.gc.runC <- struct{}{} +} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 04701ee69a..fa98848ddc 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -83,6 +83,22 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { return localStore, nil } +// isValid returns true if chunk passes any of the LocalStore Validators. +// isValid also returns true if LocalStore has no Validators. +func (ls *LocalStore) isValid(chunk Chunk) bool { + // by default chunks are valid. if we have 0 validators, then all chunks are valid. + valid := true + + // ls.Validators contains a list of one validator per chunk type. + // if one validator succeeds, then the chunk is valid + for _, v := range ls.Validators { + if valid = v.Validate(chunk.Address(), chunk.Data()); valid { + break + } + } + return valid +} + // Put is responsible for doing validation and storage of the chunk // by using configured ChunkValidators, MemStore and LDBStore. // If the chunk is not valid, its GetErrored function will @@ -96,15 +112,7 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { // After the LDBStore.Put, it is ensured that the MemStore // contains the chunk with the same data, but nil ReqC channel. func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { - valid := true - // ls.Validators contains a list of one validator per chunk type. - // if one validator succeeds, then the chunk is valid - for _, v := range ls.Validators { - if valid = v.Validate(chunk.Address(), chunk.Data()); valid { - break - } - } - if !valid { + if !ls.isValid(chunk) { return ErrChunkInvalid } @@ -145,6 +153,7 @@ func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err e if err == nil { metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) + go ls.DbStore.MarkAccessed(addr) return chunk, nil } @@ -184,3 +193,51 @@ func (ls *LocalStore) Iterator(from uint64, to uint64, po uint8, f func(Address, func (ls *LocalStore) Close() { ls.DbStore.Close() } + +// Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match +func (ls *LocalStore) Migrate() error { + actualDbSchema, err := ls.DbStore.GetSchema() + if err != nil { + log.Error(err.Error()) + return err + } + + log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) + + if actualDbSchema == CurrentDbSchema { + return nil + } + + if actualDbSchema == DbSchemaNone { + ls.migrateFromNoneToPurity() + actualDbSchema = DbSchemaPurity + } + + if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { + return err + } + + if actualDbSchema == DbSchemaPurity { + if err := ls.migrateFromPurityToHalloween(); err != nil { + return err + } + actualDbSchema = DbSchemaHalloween + } + + if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { + return err + } + return nil +} + +func (ls *LocalStore) migrateFromNoneToPurity() { + // delete chunks that are not valid, i.e. chunks that do not pass + // any of the ls.Validators + ls.DbStore.Cleanup(func(c *chunk) bool { + return !ls.isValid(c) + }) +} + +func (ls *LocalStore) migrateFromPurityToHalloween() error { + return ls.DbStore.CleanGCIndex() +} diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 814d270d38..7a07726d1c 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "testing" + "time" ch "github.com/ethereum/go-ethereum/swarm/chunk" ) @@ -30,8 +31,8 @@ var ( ) // tests that the content address validator correctly checks the data -// tests that resource update chunks are passed through content address validator -// the test checking the resouce update validator internal correctness is found in resource_test.go +// tests that feed update chunks are passed through content address validator +// the test checking the resouce update validator internal correctness is found in storage/feeds/handler_test.go func TestValidator(t *testing.T) { // set up localstore datadir, err := ioutil.TempDir("", "storage-testvalidator") @@ -144,3 +145,67 @@ func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs [] } return hs, errs } + +// TestGetFrequentlyAccessedChunkWontGetGarbageCollected tests that the most +// frequently accessed chunk is not garbage collected from LDBStore, i.e., +// from disk when we are at the capacity and garbage collector runs. For that +// we start putting random chunks into the DB while continuously accessing the +// chunk we care about then check if we can still retrieve it from disk. +func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) { + ldbCap := defaultGCRatio + store, cleanup := setupLocalStore(t, ldbCap) + defer cleanup() + + var chunks []Chunk + for i := 0; i < ldbCap; i++ { + chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize)) + } + + mostAccessed := chunks[0].Address() + for _, chunk := range chunks { + if err := store.Put(context.Background(), chunk); err != nil { + t.Fatal(err) + } + + if _, err := store.Get(context.Background(), mostAccessed); err != nil { + t.Fatal(err) + } + // Add time for MarkAccessed() to be able to finish in a separate Goroutine + time.Sleep(1 * time.Millisecond) + } + + store.DbStore.collectGarbage() + if _, err := store.DbStore.Get(context.Background(), mostAccessed); err != nil { + t.Logf("most frequntly accessed chunk not found on disk (key: %v)", mostAccessed) + t.Fatal(err) + } + +} + +func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) { + t.Helper() + + var err error + datadir, err := ioutil.TempDir("", "storage") + if err != nil { + t.Fatal(err) + } + + params := &LocalStoreParams{ + StoreParams: NewStoreParams(uint64(ldbCap), uint(ldbCap), nil, nil), + } + params.Init(datadir) + + store, err := NewLocalStore(params, nil) + if err != nil { + _ = os.RemoveAll(datadir) + t.Fatal(err) + } + + cleanup = func() { + store.Close() + _ = os.RemoveAll(datadir) + } + + return store, cleanup +} diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 6b370d2b41..6850d2d69a 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -111,7 +111,7 @@ func TestMemStoreAndLDBStore(t *testing.T) { chunkSize: 4096, }, { - n: 201, + n: 101, chunkSize: 4096, }, { @@ -119,11 +119,7 @@ func TestMemStoreAndLDBStore(t *testing.T) { chunkSize: 4096, }, { - n: 3100, - chunkSize: 4096, - }, - { - n: 100, + n: 1100, chunkSize: 4096, }, } diff --git a/swarm/storage/mock/rpc/rpc_test.go b/swarm/storage/mock/rpc/rpc_test.go index 52b634a445..f62340edec 100644 --- a/swarm/storage/mock/rpc/rpc_test.go +++ b/swarm/storage/mock/rpc/rpc_test.go @@ -37,5 +37,5 @@ func TestRPCStore(t *testing.T) { store := NewGlobalStore(rpc.DialInProc(server)) defer store.Close() - test.MockStore(t, store, 100) + test.MockStore(t, store, 30) } diff --git a/swarm/storage/mru/doc.go b/swarm/storage/mru/doc.go deleted file mode 100644 index e1d7c2c343..0000000000 --- a/swarm/storage/mru/doc.go +++ /dev/null @@ -1,61 +0,0 @@ -// Package mru defines Mutable resource updates. -// A Mutable Resource is an entity which allows updates to a resource -// without resorting to ENS on each update. -// The update scheme is built on swarm chunks with chunk keys following -// a predictable, versionable pattern. -// -// Updates are defined to be periodic in nature, where the update frequency -// is expressed in seconds. -// -// The root entry of a mutable resource is tied to a unique identifier that -// is deterministically generated out of the metadata content that describes -// the resource. This metadata includes a user-defined resource name, a resource -// start time that indicates when the resource becomes valid, -// the frequency in seconds with which the resource is expected to be updated, both of -// which are stored as little-endian uint64 values in the database (for a -// total of 16 bytes). It also contains the owner's address (ownerAddr) -// This MRU info is stored in a separate content-addressed chunk -// (call it the metadata chunk), with the following layout: -// -// (00|length|startTime|frequency|name|ownerAddr) -// -// (The two first zero-value bytes are used for disambiguation by the chunk validator, -// and update chunk will always have a value > 0 there.) -// -// Each metadata chunk is identified by its rootAddr, calculated as follows: -// metaHash=H(len(metadata), startTime, frequency,name) -// rootAddr = H(metaHash, ownerAddr). -// where H is the SHA3 hash function -// This scheme effectively locks the root chunk so that only the owner of the private key -// that ownerAddr was derived from can sign updates. -// -// The root entry tells the requester from when the mutable resource was -// first added (Unix time in seconds) and in which moments to look for the -// actual updates. Thus, a resource update for identifier "føø.bar" -// starting at unix time 1528800000 with frequency 300 (every 5 mins) will have updates on 1528800300, -// 1528800600, 1528800900 and so on. -// -// Actual data updates are also made in the form of swarm chunks. The keys -// of the updates are the hash of a concatenation of properties as follows: -// -// updateAddr = H(period, version, rootAddr) -// where H is the SHA3 hash function -// The period is (currentTime - startTime) / frequency -// -// Using our previous example, this means that a period 3 will happen when the -// clock hits 1528800900 -// -// If more than one update is made in the same period, incremental -// version numbers are used successively. -// -// A user looking up a resource would only need to know the rootAddr in order to get the versions -// -// the resource update data is: -// resourcedata = headerlength|period|version|rootAddr|flags|metaHash -// where flags is a 1-byte flags field. Flag 0 is set to 1 to indicate multihash -// -// the full update data that goes in the chunk payload is: -// resourcedata|sign(resourcedata) -// -// headerlength is a 16 bit value containing the byte length of period|version|rootAddr|flags|metaHash -package mru diff --git a/swarm/storage/mru/handler.go b/swarm/storage/mru/handler.go deleted file mode 100644 index 18c667f14e..0000000000 --- a/swarm/storage/mru/handler.go +++ /dev/null @@ -1,516 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Handler is the API for Mutable Resources -// It enables creating, updating, syncing and retrieving resources and their update data -package mru - -import ( - "bytes" - "context" - "sync" - "time" - "unsafe" - - "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/storage" -) - -type Handler struct { - chunkStore *storage.NetStore - HashSize int - resources map[uint64]*resource - resourceLock sync.RWMutex - storeTimeout time.Duration - queryMaxPeriods uint32 -} - -// HandlerParams pass parameters to the Handler constructor NewHandler -// Signer and TimestampProvider are mandatory parameters -type HandlerParams struct { - QueryMaxPeriods uint32 -} - -// hashPool contains a pool of ready hashers -var hashPool sync.Pool -var minimumChunkLength int - -// init initializes the package and hashPool -func init() { - hashPool = sync.Pool{ - New: func() interface{} { - return storage.MakeHashFunc(resourceHashAlgorithm)() - }, - } - if minimumMetadataLength < minimumUpdateDataLength { - minimumChunkLength = minimumMetadataLength - } else { - minimumChunkLength = minimumUpdateDataLength - } -} - -// NewHandler creates a new Mutable Resource API -func NewHandler(params *HandlerParams) *Handler { - rh := &Handler{ - resources: make(map[uint64]*resource), - storeTimeout: defaultStoreTimeout, - queryMaxPeriods: params.QueryMaxPeriods, - } - - for i := 0; i < hasherCount; i++ { - hashfunc := storage.MakeHashFunc(resourceHashAlgorithm)() - if rh.HashSize == 0 { - rh.HashSize = hashfunc.Size() - } - hashPool.Put(hashfunc) - } - - return rh -} - -// SetStore sets the store backend for the Mutable Resource API -func (h *Handler) SetStore(store *storage.NetStore) { - h.chunkStore = store -} - -// Validate is a chunk validation method -// If it looks like a resource update, the chunk address is checked against the ownerAddr of the update's signature -// It implements the storage.ChunkValidator interface -func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { - dataLength := len(data) - if dataLength < minimumChunkLength || dataLength > chunk.DefaultSize+8 { - return false - } - - //metadata chunks have the first two bytes set to zero - if data[0] == 0 && data[1] == 0 && dataLength >= minimumMetadataLength { - //metadata chunk - rootAddr, _ := metadataHash(data) - valid := bytes.Equal(chunkAddr, rootAddr) - if !valid { - log.Debug("Invalid root metadata chunk with address", "addr", chunkAddr.Hex()) - } - return valid - } - - // if it is not a metadata chunk, check if it is a properly formatted update chunk with - // valid signature and proof of ownership of the resource it is trying - // to update - - // First, deserialize the chunk - var r SignedResourceUpdate - if err := r.fromChunk(chunkAddr, data); err != nil { - log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error()) - return false - } - - // check that the lookup information contained in the chunk matches the updateAddr (chunk search key) - // that was used to retrieve this chunk - // if this validation fails, someone forged a chunk. - if !bytes.Equal(chunkAddr, r.updateHeader.UpdateAddr()) { - log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr", "addr", chunkAddr.Hex()) - return false - } - - // Verify signatures and that the signer actually owns the resource - // If it fails, it means either the signature is not valid, data is corrupted - // or someone is trying to update someone else's resource. - if err := r.Verify(); err != nil { - log.Debug("Invalid signature", "err", err) - return false - } - - return true -} - -// GetContent retrieves the data payload of the last synced update of the Mutable Resource -func (h *Handler) GetContent(rootAddr storage.Address) (storage.Address, []byte, error) { - rsrc := h.get(rootAddr) - if rsrc == nil || !rsrc.isSynced() { - return nil, nil, NewError(ErrNotFound, " does not exist or is not synced") - } - return rsrc.lastKey, rsrc.data, nil -} - -// GetLastPeriod retrieves the period of the last synced update of the Mutable Resource -func (h *Handler) GetLastPeriod(rootAddr storage.Address) (uint32, error) { - rsrc := h.get(rootAddr) - if rsrc == nil { - return 0, NewError(ErrNotFound, " does not exist") - } else if !rsrc.isSynced() { - return 0, NewError(ErrNotSynced, " is not synced") - } - return rsrc.period, nil -} - -// GetVersion retrieves the period of the last synced update of the Mutable Resource -func (h *Handler) GetVersion(rootAddr storage.Address) (uint32, error) { - rsrc := h.get(rootAddr) - if rsrc == nil { - return 0, NewError(ErrNotFound, " does not exist") - } else if !rsrc.isSynced() { - return 0, NewError(ErrNotSynced, " is not synced") - } - return rsrc.version, nil -} - -// New creates a new metadata chunk out of the request passed in. -func (h *Handler) New(ctx context.Context, request *Request) error { - - // frequency 0 is invalid - if request.metadata.Frequency == 0 { - return NewError(ErrInvalidValue, "frequency cannot be 0 when creating a resource") - } - - // make sure owner is set to something - if request.metadata.Owner == zeroAddr { - return NewError(ErrInvalidValue, "ownerAddr must be set to create a new metadata chunk") - } - - // create the meta chunk and store it in swarm - chunk, metaHash, err := request.metadata.newChunk() - if err != nil { - return err - } - if request.metaHash != nil && !bytes.Equal(request.metaHash, metaHash) || - request.rootAddr != nil && !bytes.Equal(request.rootAddr, chunk.Address()) { - return NewError(ErrInvalidValue, "metaHash in UpdateRequest does not match actual metadata") - } - - request.metaHash = metaHash - request.rootAddr = chunk.Address() - - h.chunkStore.Put(ctx, chunk) - log.Debug("new resource", "name", request.metadata.Name, "startTime", request.metadata.StartTime, "frequency", request.metadata.Frequency, "owner", request.metadata.Owner) - - // create the internal index for the resource and populate it with its metadata - rsrc := &resource{ - resourceUpdate: resourceUpdate{ - updateHeader: updateHeader{ - UpdateLookup: UpdateLookup{ - rootAddr: chunk.Address(), - }, - }, - }, - ResourceMetadata: request.metadata, - updated: time.Now(), - } - h.set(chunk.Address(), rsrc) - - return nil -} - -// NewUpdateRequest prepares an UpdateRequest structure with all the necessary information to -// just add the desired data and sign it. -// The resulting structure can then be signed and passed to Handler.Update to be verified and sent -func (h *Handler) NewUpdateRequest(ctx context.Context, rootAddr storage.Address) (updateRequest *Request, err error) { - - if rootAddr == nil { - return nil, NewError(ErrInvalidValue, "rootAddr cannot be nil") - } - - // Make sure we have a cache of the metadata chunk - rsrc, err := h.Load(ctx, rootAddr) - if err != nil { - return nil, err - } - - now := TimestampProvider.Now() - - updateRequest = new(Request) - updateRequest.period, err = getNextPeriod(rsrc.StartTime.Time, now.Time, rsrc.Frequency) - if err != nil { - return nil, err - } - - if _, err = h.lookup(rsrc, LookupLatestVersionInPeriod(rsrc.rootAddr, updateRequest.period)); err != nil { - if err.(*Error).code != ErrNotFound { - return nil, err - } - // not finding updates means that there is a network error - // or that the resource really does not have updates in this period. - } - - updateRequest.multihash = rsrc.multihash - updateRequest.rootAddr = rsrc.rootAddr - updateRequest.metaHash = rsrc.metaHash - updateRequest.metadata = rsrc.ResourceMetadata - - // if we already have an update for this period then increment version - // resource object MUST be in sync for version to be correct, but we checked this earlier in the method already - if h.hasUpdate(rootAddr, updateRequest.period) { - updateRequest.version = rsrc.version + 1 - } else { - updateRequest.version = 1 - } - - return updateRequest, nil -} - -// Lookup retrieves a specific or latest version of the resource update with metadata chunk at params.Root -// Lookup works differently depending on the configuration of `LookupParams` -// See the `LookupParams` documentation and helper functions: -// `LookupLatest`, `LookupLatestVersionInPeriod` and `LookupVersion` -// When looking for the latest update, it starts at the next period after the current time. -// upon failure tries the corresponding keys of each previous period until one is found -// (or startTime is reached, in which case there are no updates). -func (h *Handler) Lookup(ctx context.Context, params *LookupParams) (*resource, error) { - - rsrc := h.get(params.rootAddr) - if rsrc == nil { - return nil, NewError(ErrNothingToReturn, "resource not loaded") - } - return h.lookup(rsrc, params) -} - -// LookupPrevious returns the resource before the one currently loaded in the resource cache -// This is useful where resource updates are used incrementally in contrast to -// merely replacing content. -// Requires a cached resource object to determine the current state of the resource. -func (h *Handler) LookupPrevious(ctx context.Context, params *LookupParams) (*resource, error) { - rsrc := h.get(params.rootAddr) - if rsrc == nil { - return nil, NewError(ErrNothingToReturn, "resource not loaded") - } - if !rsrc.isSynced() { - return nil, NewError(ErrNotSynced, "LookupPrevious requires synced resource.") - } else if rsrc.period == 0 { - return nil, NewError(ErrNothingToReturn, " not found") - } - var version, period uint32 - if rsrc.version > 1 { - version = rsrc.version - 1 - period = rsrc.period - } else if rsrc.period == 1 { - return nil, NewError(ErrNothingToReturn, "Current update is the oldest") - } else { - version = 0 - period = rsrc.period - 1 - } - return h.lookup(rsrc, NewLookupParams(rsrc.rootAddr, period, version, params.Limit)) -} - -// base code for public lookup methods -func (h *Handler) lookup(rsrc *resource, params *LookupParams) (*resource, error) { - - lp := *params - // we can't look for anything without a store - if h.chunkStore == nil { - return nil, NewError(ErrInit, "Call Handler.SetStore() before performing lookups") - } - - var specificperiod bool - if lp.period > 0 { - specificperiod = true - } else { - // get the current time and the next period - now := TimestampProvider.Now() - - var period uint32 - period, err := getNextPeriod(rsrc.StartTime.Time, now.Time, rsrc.Frequency) - if err != nil { - return nil, err - } - lp.period = period - } - - // start from the last possible period, and iterate previous ones - // (unless we want a specific period only) until we find a match. - // If we hit startTime we're out of options - var specificversion bool - if lp.version > 0 { - specificversion = true - } else { - lp.version = 1 - } - - var hops uint32 - if lp.Limit == 0 { - lp.Limit = h.queryMaxPeriods - } - log.Trace("resource lookup", "period", lp.period, "version", lp.version, "limit", lp.Limit) - for lp.period > 0 { - if lp.Limit != 0 && hops > lp.Limit { - return nil, NewErrorf(ErrPeriodDepth, "Lookup exceeded max period hops (%d)", lp.Limit) - } - updateAddr := lp.UpdateAddr() - - ctx, cancel := context.WithTimeout(context.Background(), defaultRetrieveTimeout) - defer cancel() - - chunk, err := h.chunkStore.Get(ctx, updateAddr) - if err == nil { - if specificversion { - return h.updateIndex(rsrc, chunk) - } - // check if we have versions > 1. If a version fails, the previous version is used and returned. - log.Trace("rsrc update version 1 found, checking for version updates", "period", lp.period, "updateAddr", updateAddr) - for { - newversion := lp.version + 1 - updateAddr := lp.UpdateAddr() - - ctx, cancel := context.WithTimeout(context.Background(), defaultRetrieveTimeout) - defer cancel() - - newchunk, err := h.chunkStore.Get(ctx, updateAddr) - if err != nil { - return h.updateIndex(rsrc, chunk) - } - chunk = newchunk - lp.version = newversion - log.Trace("version update found, checking next", "version", lp.version, "period", lp.period, "updateAddr", updateAddr) - } - } - if specificperiod { - break - } - log.Trace("rsrc update not found, checking previous period", "period", lp.period, "updateAddr", updateAddr) - lp.period-- - hops++ - } - return nil, NewError(ErrNotFound, "no updates found") -} - -// Load retrieves the Mutable Resource metadata chunk stored at rootAddr -// Upon retrieval it creates/updates the index entry for it with metadata corresponding to the chunk contents -func (h *Handler) Load(ctx context.Context, rootAddr storage.Address) (*resource, error) { - //TODO: Maybe add timeout to context, defaultRetrieveTimeout? - ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout) - defer cancel() - chunk, err := h.chunkStore.Get(ctx, rootAddr) - if err != nil { - return nil, NewError(ErrNotFound, err.Error()) - } - - // create the index entry - rsrc := &resource{} - - if err := rsrc.ResourceMetadata.binaryGet(chunk.Data()); err != nil { // Will fail if this is not really a metadata chunk - return nil, err - } - - rsrc.rootAddr, rsrc.metaHash = metadataHash(chunk.Data()) - if !bytes.Equal(rsrc.rootAddr, rootAddr) { - return nil, NewError(ErrCorruptData, "Corrupt metadata chunk") - } - h.set(rootAddr, rsrc) - log.Trace("resource index load", "rootkey", rootAddr, "name", rsrc.ResourceMetadata.Name, "starttime", rsrc.ResourceMetadata.StartTime, "frequency", rsrc.ResourceMetadata.Frequency) - return rsrc, nil -} - -// update mutable resource index map with specified content -func (h *Handler) updateIndex(rsrc *resource, chunk storage.Chunk) (*resource, error) { - - // retrieve metadata from chunk data and check that it matches this mutable resource - var r SignedResourceUpdate - if err := r.fromChunk(chunk.Address(), chunk.Data()); err != nil { - return nil, err - } - log.Trace("resource index update", "name", rsrc.ResourceMetadata.Name, "updatekey", chunk.Address(), "period", r.period, "version", r.version) - - // update our rsrcs entry map - rsrc.lastKey = chunk.Address() - rsrc.period = r.period - rsrc.version = r.version - rsrc.updated = time.Now() - rsrc.data = make([]byte, len(r.data)) - rsrc.multihash = r.multihash - copy(rsrc.data, r.data) - rsrc.Reader = bytes.NewReader(rsrc.data) - log.Debug("resource synced", "name", rsrc.ResourceMetadata.Name, "updateAddr", chunk.Address(), "period", rsrc.period, "version", rsrc.version) - h.set(chunk.Address(), rsrc) - return rsrc, nil -} - -// Update adds an actual data update -// Uses the Mutable Resource metadata currently loaded in the resources map entry. -// It is the caller's responsibility to make sure that this data is not stale. -// Note that a Mutable Resource update cannot span chunks, and thus has a MAX NET LENGTH 4096, INCLUDING update header data and signature. An error will be returned if the total length of the chunk payload will exceed this limit. -// Update can only check if the caller is trying to overwrite the very last known version, otherwise it just puts the update -// on the network. -func (h *Handler) Update(ctx context.Context, r *SignedResourceUpdate) (storage.Address, error) { - return h.update(ctx, r) -} - -// create and commit an update -func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAddr storage.Address, err error) { - - // we can't update anything without a store - if h.chunkStore == nil { - return nil, NewError(ErrInit, "Call Handler.SetStore() before updating") - } - - rsrc := h.get(r.rootAddr) - if rsrc != nil && rsrc.period != 0 && rsrc.version != 0 && // This is the only cheap check we can do for sure - rsrc.period == r.period && rsrc.version >= r.version { // without having to lookup update chunks - - return nil, NewError(ErrInvalidValue, "A former update in this period is already known to exist") - } - - chunk, err := r.toChunk() // Serialize the update into a chunk. Fails if data is too big - if err != nil { - return nil, err - } - - // send the chunk - h.chunkStore.Put(ctx, chunk) - log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.Data(), "multihash", r.multihash) - - // update our resources map entry if the new update is older than the one we have, if we have it. - if rsrc != nil && (r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version)) { - rsrc.period = r.period - rsrc.version = r.version - rsrc.data = make([]byte, len(r.data)) - rsrc.updated = time.Now() - rsrc.lastKey = r.updateAddr - rsrc.multihash = r.multihash - copy(rsrc.data, r.data) - rsrc.Reader = bytes.NewReader(rsrc.data) - } - return r.updateAddr, nil -} - -// Retrieves the resource index value for the given nameHash -func (h *Handler) get(rootAddr storage.Address) *resource { - if len(rootAddr) < storage.AddressLength { - log.Warn("Handler.get with invalid rootAddr") - return nil - } - hashKey := *(*uint64)(unsafe.Pointer(&rootAddr[0])) - h.resourceLock.RLock() - defer h.resourceLock.RUnlock() - rsrc := h.resources[hashKey] - return rsrc -} - -// Sets the resource index value for the given nameHash -func (h *Handler) set(rootAddr storage.Address, rsrc *resource) { - if len(rootAddr) < storage.AddressLength { - log.Warn("Handler.set with invalid rootAddr") - return - } - hashKey := *(*uint64)(unsafe.Pointer(&rootAddr[0])) - h.resourceLock.Lock() - defer h.resourceLock.Unlock() - h.resources[hashKey] = rsrc -} - -// Checks if we already have an update on this resource, according to the value in the current state of the resource index -func (h *Handler) hasUpdate(rootAddr storage.Address, period uint32) bool { - rsrc := h.get(rootAddr) - return rsrc != nil && rsrc.period == period -} diff --git a/swarm/storage/mru/lookup.go b/swarm/storage/mru/lookup.go deleted file mode 100644 index b52cd5b4ff..0000000000 --- a/swarm/storage/mru/lookup.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "encoding/binary" - "hash" - - "github.com/ethereum/go-ethereum/swarm/storage" -) - -// LookupParams is used to specify constraints when performing an update lookup -// Limit defines whether or not the lookup should be limited -// If Limit is set to true then Max defines the amount of hops that can be performed -type LookupParams struct { - UpdateLookup - Limit uint32 -} - -// RootAddr returns the metadata chunk address -func (r *LookupParams) RootAddr() storage.Address { - return r.rootAddr -} - -func NewLookupParams(rootAddr storage.Address, period, version uint32, limit uint32) *LookupParams { - return &LookupParams{ - UpdateLookup: UpdateLookup{ - period: period, - version: version, - rootAddr: rootAddr, - }, - Limit: limit, - } -} - -// LookupLatest generates lookup parameters that look for the latest version of a resource -func LookupLatest(rootAddr storage.Address) *LookupParams { - return NewLookupParams(rootAddr, 0, 0, 0) -} - -// LookupLatestVersionInPeriod generates lookup parameters that look for the latest version of a resource in a given period -func LookupLatestVersionInPeriod(rootAddr storage.Address, period uint32) *LookupParams { - return NewLookupParams(rootAddr, period, 0, 0) -} - -// LookupVersion generates lookup parameters that look for a specific version of a resource -func LookupVersion(rootAddr storage.Address, period, version uint32) *LookupParams { - return NewLookupParams(rootAddr, period, version, 0) -} - -// UpdateLookup represents the components of a resource update search key -type UpdateLookup struct { - period uint32 - version uint32 - rootAddr storage.Address -} - -// 4 bytes period -// 4 bytes version -// storage.Keylength for rootAddr -const updateLookupLength = 4 + 4 + storage.AddressLength - -// UpdateAddr calculates the resource update chunk address corresponding to this lookup key -func (u *UpdateLookup) UpdateAddr() (updateAddr storage.Address) { - serializedData := make([]byte, updateLookupLength) - u.binaryPut(serializedData) - hasher := hashPool.Get().(hash.Hash) - defer hashPool.Put(hasher) - hasher.Reset() - hasher.Write(serializedData) - return hasher.Sum(nil) -} - -// binaryPut serializes this UpdateLookup instance into the provided slice -func (u *UpdateLookup) binaryPut(serializedData []byte) error { - if len(serializedData) != updateLookupLength { - return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize UpdateLookup. Expected %d, got %d", updateLookupLength, len(serializedData)) - } - if len(u.rootAddr) != storage.AddressLength { - return NewError(ErrInvalidValue, "UpdateLookup.binaryPut called without rootAddr set") - } - binary.LittleEndian.PutUint32(serializedData[:4], u.period) - binary.LittleEndian.PutUint32(serializedData[4:8], u.version) - copy(serializedData[8:], u.rootAddr[:]) - return nil -} - -// binaryLength returns the expected size of this structure when serialized -func (u *UpdateLookup) binaryLength() int { - return updateLookupLength -} - -// binaryGet restores the current instance from the information contained in the passed slice -func (u *UpdateLookup) binaryGet(serializedData []byte) error { - if len(serializedData) != updateLookupLength { - return NewErrorf(ErrInvalidValue, "Incorrect slice size to read UpdateLookup. Expected %d, got %d", updateLookupLength, len(serializedData)) - } - u.period = binary.LittleEndian.Uint32(serializedData[:4]) - u.version = binary.LittleEndian.Uint32(serializedData[4:8]) - u.rootAddr = storage.Address(make([]byte, storage.AddressLength)) - copy(u.rootAddr[:], serializedData[8:]) - return nil -} diff --git a/swarm/storage/mru/lookup_test.go b/swarm/storage/mru/lookup_test.go deleted file mode 100644 index b66b200a36..0000000000 --- a/swarm/storage/mru/lookup_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package mru - -import ( - "bytes" - "testing" - - "github.com/ethereum/go-ethereum/common/hexutil" -) - -func getTestUpdateLookup() *UpdateLookup { - metadata := *getTestMetadata() - rootAddr, _, _, _ := metadata.serializeAndHash() - return &UpdateLookup{ - period: 79, - version: 2010, - rootAddr: rootAddr, - } -} - -func compareUpdateLookup(a, b *UpdateLookup) bool { - return a.version == b.version && - a.period == b.period && - bytes.Equal(a.rootAddr, b.rootAddr) -} - -func TestUpdateLookupUpdateAddr(t *testing.T) { - ul := getTestUpdateLookup() - updateAddr := ul.UpdateAddr() - compareByteSliceToExpectedHex(t, "updateAddr", updateAddr, "0x8fbc8d4777ef6da790257eda80ab4321fabd08cbdbe67e4e3da6caca386d64e0") -} - -func TestUpdateLookupSerializer(t *testing.T) { - serializedUpdateLookup := make([]byte, updateLookupLength) - ul := getTestUpdateLookup() - if err := ul.binaryPut(serializedUpdateLookup); err != nil { - t.Fatal(err) - } - compareByteSliceToExpectedHex(t, "serializedUpdateLookup", serializedUpdateLookup, "0x4f000000da070000fb0ed7efa696bdb0b54cd75554cc3117ffc891454317df7dd6fefad978e2f2fb") - - // set receiving slice to the wrong size - serializedUpdateLookup = make([]byte, updateLookupLength+7) - if err := ul.binaryPut(serializedUpdateLookup); err == nil { - t.Fatalf("Expected UpdateLookup.binaryPut to fail when receiving slice has a length != %d", updateLookupLength) - } - - // set rootAddr to an invalid length - ul.rootAddr = []byte{1, 2, 3, 4} - serializedUpdateLookup = make([]byte, updateLookupLength) - if err := ul.binaryPut(serializedUpdateLookup); err == nil { - t.Fatal("Expected UpdateLookup.binaryPut to fail when rootAddr is not of the correct size") - } -} - -func TestUpdateLookupDeserializer(t *testing.T) { - serializedUpdateLookup, _ := hexutil.Decode("0x4f000000da070000fb0ed7efa696bdb0b54cd75554cc3117ffc891454317df7dd6fefad978e2f2fb") - var recoveredUpdateLookup UpdateLookup - if err := recoveredUpdateLookup.binaryGet(serializedUpdateLookup); err != nil { - t.Fatal(err) - } - originalUpdateLookup := *getTestUpdateLookup() - if !compareUpdateLookup(&originalUpdateLookup, &recoveredUpdateLookup) { - t.Fatalf("Expected recovered UpdateLookup to match") - } - - // set source slice to the wrong size - serializedUpdateLookup = make([]byte, updateLookupLength+4) - if err := recoveredUpdateLookup.binaryGet(serializedUpdateLookup); err == nil { - t.Fatalf("Expected UpdateLookup.binaryGet to fail when source slice has a length != %d", updateLookupLength) - } -} - -func TestUpdateLookupSerializeDeserialize(t *testing.T) { - serializedUpdateLookup := make([]byte, updateLookupLength) - originalUpdateLookup := getTestUpdateLookup() - if err := originalUpdateLookup.binaryPut(serializedUpdateLookup); err != nil { - t.Fatal(err) - } - var recoveredUpdateLookup UpdateLookup - if err := recoveredUpdateLookup.binaryGet(serializedUpdateLookup); err != nil { - t.Fatal(err) - } - if !compareUpdateLookup(originalUpdateLookup, &recoveredUpdateLookup) { - t.Fatalf("Expected recovered UpdateLookup to match") - } -} diff --git a/swarm/storage/mru/metadata.go b/swarm/storage/mru/metadata.go deleted file mode 100644 index 5091148959..0000000000 --- a/swarm/storage/mru/metadata.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "encoding/binary" - "hash" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/storage" -) - -// ResourceMetadata encapsulates the immutable information about a mutable resource :) -// once serialized into a chunk, the resource can be retrieved by knowing its content-addressed rootAddr -type ResourceMetadata struct { - StartTime Timestamp // time at which the resource starts to be valid - Frequency uint64 // expected update frequency for the resource - Name string // name of the resource, for the reference of the user or to disambiguate resources with same starttime, frequency, owneraddr - Owner common.Address // public address of the resource owner -} - -const frequencyLength = 8 // sizeof(uint64) -const nameLengthLength = 1 - -// Resource metadata chunk layout: -// 4 prefix bytes (chunkPrefixLength). The first two set to zero. The second two indicate the length -// Timestamp: timestampLength bytes -// frequency: frequencyLength bytes -// name length: nameLengthLength bytes -// name (variable length, can be empty, up to 255 bytes) -// ownerAddr: common.AddressLength -const minimumMetadataLength = chunkPrefixLength + timestampLength + frequencyLength + nameLengthLength + common.AddressLength - -// binaryGet populates the resource metadata from a byte array -func (r *ResourceMetadata) binaryGet(serializedData []byte) error { - if len(serializedData) < minimumMetadataLength { - return NewErrorf(ErrInvalidValue, "Metadata chunk to deserialize is too short. Expected at least %d. Got %d.", minimumMetadataLength, len(serializedData)) - } - - // first two bytes must be set to zero to indicate metadata chunks, so enforce this. - if serializedData[0] != 0 || serializedData[1] != 0 { - return NewError(ErrCorruptData, "Invalid metadata chunk") - } - - cursor := 2 - metadataLength := int(binary.LittleEndian.Uint16(serializedData[cursor : cursor+2])) // metadataLength does not include the 4 prefix bytes - if metadataLength+chunkPrefixLength != len(serializedData) { - return NewErrorf(ErrCorruptData, "Incorrect declared metadata length. Expected %d, got %d.", metadataLength+chunkPrefixLength, len(serializedData)) - } - - cursor += 2 - - if err := r.StartTime.binaryGet(serializedData[cursor : cursor+timestampLength]); err != nil { - return err - } - cursor += timestampLength - - r.Frequency = binary.LittleEndian.Uint64(serializedData[cursor : cursor+frequencyLength]) - cursor += frequencyLength - - nameLength := int(serializedData[cursor]) - if nameLength+minimumMetadataLength > len(serializedData) { - return NewErrorf(ErrInvalidValue, "Metadata chunk to deserialize is too short when decoding resource name. Expected at least %d. Got %d.", nameLength+minimumMetadataLength, len(serializedData)) - } - cursor++ - r.Name = string(serializedData[cursor : cursor+nameLength]) - cursor += nameLength - - copy(r.Owner[:], serializedData[cursor:]) - cursor += common.AddressLength - if cursor != len(serializedData) { - return NewErrorf(ErrInvalidValue, "Metadata chunk has leftover data after deserialization. %d left to read", len(serializedData)-cursor) - } - return nil -} - -// binaryPut encodes the metadata into a byte array -func (r *ResourceMetadata) binaryPut(serializedData []byte) error { - metadataChunkLength := r.binaryLength() - if len(serializedData) != metadataChunkLength { - return NewErrorf(ErrInvalidValue, "Need a slice of exactly %d bytes to serialize this metadata, but got a slice of size %d.", metadataChunkLength, len(serializedData)) - } - - // root chunk has first two bytes both set to 0, which distinguishes from update bytes - // therefore, skip the first two bytes of a zero-initialized array. - cursor := 2 - binary.LittleEndian.PutUint16(serializedData[cursor:cursor+2], uint16(metadataChunkLength-chunkPrefixLength)) // metadataLength does not include the 4 prefix bytes - cursor += 2 - - r.StartTime.binaryPut(serializedData[cursor : cursor+timestampLength]) - cursor += timestampLength - - binary.LittleEndian.PutUint64(serializedData[cursor:cursor+frequencyLength], r.Frequency) - cursor += frequencyLength - - // Encode the name string as a 1 byte length followed by the encoded string. - // Longer strings will be truncated. - nameLength := len(r.Name) - if nameLength > 255 { - nameLength = 255 - } - serializedData[cursor] = uint8(nameLength) - cursor++ - copy(serializedData[cursor:cursor+nameLength], []byte(r.Name[:nameLength])) - cursor += nameLength - - copy(serializedData[cursor:cursor+common.AddressLength], r.Owner[:]) - cursor += common.AddressLength - - return nil -} - -func (r *ResourceMetadata) binaryLength() int { - return minimumMetadataLength + len(r.Name) -} - -// serializeAndHash returns the root chunk addr and metadata hash that help identify and ascertain ownership of this resource -// returns the serialized metadata as a byproduct of having to hash it. -func (r *ResourceMetadata) serializeAndHash() (rootAddr, metaHash []byte, chunkData []byte, err error) { - - chunkData = make([]byte, r.binaryLength()) - if err := r.binaryPut(chunkData); err != nil { - return nil, nil, nil, err - } - rootAddr, metaHash = metadataHash(chunkData) - return rootAddr, metaHash, chunkData, nil - -} - -// creates a metadata chunk out of a resourceMetadata structure -func (metadata *ResourceMetadata) newChunk() (chunk storage.Chunk, metaHash []byte, err error) { - // the metadata chunk contains a timestamp of when the resource starts to be valid - // and also how frequently it is expected to be updated - // from this we know at what time we should look for updates, and how often - // it also contains the name of the resource, so we know what resource we are working with - - // the key (rootAddr) of the metadata chunk is content-addressed - // if it wasn't we couldn't replace it later - // resolving this relationship is left up to external agents (for example ENS) - rootAddr, metaHash, chunkData, err := metadata.serializeAndHash() - if err != nil { - return nil, nil, err - } - - // make the chunk and send it to swarm - chunk = storage.NewChunk(rootAddr, chunkData) - - return chunk, metaHash, nil -} - -// metadataHash returns the metadata chunk root address and metadata hash -// that help identify and ascertain ownership of this resource -// We compute it as rootAddr = H(ownerAddr, H(metadata)) -// Where H() is SHA3 -// metadata are all the metadata fields, except ownerAddr -// ownerAddr is the public address of the resource owner -// Update chunks must carry a rootAddr reference and metaHash in order to be verified -// This way, a node that receives an update can check the signature, recover the public address -// and check the ownership by computing H(ownerAddr, metaHash) and comparing it to the rootAddr -// the resource is claiming to update without having to lookup the metadata chunk. -// see verifyResourceOwnerhsip in signedupdate.go -func metadataHash(chunkData []byte) (rootAddr, metaHash []byte) { - hasher := hashPool.Get().(hash.Hash) - defer hashPool.Put(hasher) - hasher.Reset() - hasher.Write(chunkData[:len(chunkData)-common.AddressLength]) - metaHash = hasher.Sum(nil) - hasher.Reset() - hasher.Write(metaHash) - hasher.Write(chunkData[len(chunkData)-common.AddressLength:]) - rootAddr = hasher.Sum(nil) - return -} diff --git a/swarm/storage/mru/metadata_test.go b/swarm/storage/mru/metadata_test.go deleted file mode 100644 index abbac6e3e4..0000000000 --- a/swarm/storage/mru/metadata_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . -package mru - -import ( - "testing" - - "github.com/ethereum/go-ethereum/common/hexutil" -) - -func compareByteSliceToExpectedHex(t *testing.T, variableName string, actualValue []byte, expectedHex string) { - if hexutil.Encode(actualValue) != expectedHex { - t.Fatalf("%s: Expected %s to be %s, got %s", t.Name(), variableName, expectedHex, hexutil.Encode(actualValue)) - } -} - -func getTestMetadata() *ResourceMetadata { - return &ResourceMetadata{ - Name: "world news report, every hour, on the hour", - StartTime: Timestamp{ - Time: 1528880400, - }, - Frequency: 3600, - Owner: newCharlieSigner().Address(), - } -} - -func TestMetadataSerializerDeserializer(t *testing.T) { - metadata := *getTestMetadata() - - rootAddr, metaHash, chunkData, err := metadata.serializeAndHash() // creates hashes and marshals, in one go - if err != nil { - t.Fatal(err) - } - const expectedRootAddr = "0xfb0ed7efa696bdb0b54cd75554cc3117ffc891454317df7dd6fefad978e2f2fb" - const expectedMetaHash = "0xf74a10ce8f26ffc8bfaa07c3031a34b2c61f517955e7deb1592daccf96c69cf0" - const expectedChunkData = "0x00004f0010dd205b00000000100e0000000000002a776f726c64206e657773207265706f72742c20657665727920686f75722c206f6e2074686520686f7572876a8936a7cd0b79ef0735ad0896c1afe278781c" - - compareByteSliceToExpectedHex(t, "rootAddr", rootAddr, expectedRootAddr) - compareByteSliceToExpectedHex(t, "metaHash", metaHash, expectedMetaHash) - compareByteSliceToExpectedHex(t, "chunkData", chunkData, expectedChunkData) - - recoveredMetadata := ResourceMetadata{} - recoveredMetadata.binaryGet(chunkData) - - if recoveredMetadata != metadata { - t.Fatalf("Expected that the recovered metadata equals the marshalled metadata") - } - - // we are going to mess with the data, so create a backup to go back to it for the next test - backup := make([]byte, len(chunkData)) - copy(backup, chunkData) - - chunkData = []byte{1, 2, 3} - if err := recoveredMetadata.binaryGet(chunkData); err == nil { - t.Fatal("Expected binaryGet to fail since chunk is too small") - } - - // restore backup - chunkData = make([]byte, len(backup)) - copy(chunkData, backup) - - // mess with the prefix so it is not zero - chunkData[0] = 7 - chunkData[1] = 9 - - if err := recoveredMetadata.binaryGet(chunkData); err == nil { - t.Fatal("Expected binaryGet to fail since prefix bytes are not zero") - } - - // restore backup - chunkData = make([]byte, len(backup)) - copy(chunkData, backup) - - // mess with the length header to trigger an error - chunkData[2] = 255 - chunkData[3] = 44 - if err := recoveredMetadata.binaryGet(chunkData); err == nil { - t.Fatal("Expected binaryGet to fail since header length does not match") - } - - // restore backup - chunkData = make([]byte, len(backup)) - copy(chunkData, backup) - - // mess with name length header to trigger a chunk too short error - chunkData[20] = 255 - if err := recoveredMetadata.binaryGet(chunkData); err == nil { - t.Fatal("Expected binaryGet to fail since name length is incorrect") - } - - // restore backup - chunkData = make([]byte, len(backup)) - copy(chunkData, backup) - - // mess with name length header to trigger an leftover bytes to read error - chunkData[20] = 3 - if err := recoveredMetadata.binaryGet(chunkData); err == nil { - t.Fatal("Expected binaryGet to fail since name length is too small") - } -} - -func TestMetadataSerializerLengthCheck(t *testing.T) { - metadata := *getTestMetadata() - - // make a slice that is too small to contain the metadata - serializedMetadata := make([]byte, 4) - - if err := metadata.binaryPut(serializedMetadata); err == nil { - t.Fatal("Expected metadata.binaryPut to fail, since target slice is too small") - } - -} diff --git a/swarm/storage/mru/request.go b/swarm/storage/mru/request.go deleted file mode 100644 index af2ccf5c76..0000000000 --- a/swarm/storage/mru/request.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "bytes" - "encoding/json" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/swarm/storage" -) - -// updateRequestJSON represents a JSON-serialized UpdateRequest -type updateRequestJSON struct { - Name string `json:"name,omitempty"` - Frequency uint64 `json:"frequency,omitempty"` - StartTime uint64 `json:"startTime,omitempty"` - Owner string `json:"ownerAddr,omitempty"` - RootAddr string `json:"rootAddr,omitempty"` - MetaHash string `json:"metaHash,omitempty"` - Version uint32 `json:"version,omitempty"` - Period uint32 `json:"period,omitempty"` - Data string `json:"data,omitempty"` - Multihash bool `json:"multiHash"` - Signature string `json:"signature,omitempty"` -} - -// Request represents an update and/or resource create message -type Request struct { - SignedResourceUpdate - metadata ResourceMetadata - isNew bool -} - -var zeroAddr = common.Address{} - -// NewCreateUpdateRequest returns a ready to sign request to create and initialize a resource with data -func NewCreateUpdateRequest(metadata *ResourceMetadata) (*Request, error) { - - request, err := NewCreateRequest(metadata) - if err != nil { - return nil, err - } - - // get the current time - now := TimestampProvider.Now().Time - - request.version = 1 - request.period, err = getNextPeriod(metadata.StartTime.Time, now, metadata.Frequency) - if err != nil { - return nil, err - } - return request, nil -} - -// NewCreateRequest returns a request to create a new resource -func NewCreateRequest(metadata *ResourceMetadata) (request *Request, err error) { - if metadata.StartTime.Time == 0 { // get the current time - metadata.StartTime = TimestampProvider.Now() - } - - if metadata.Owner == zeroAddr { - return nil, NewError(ErrInvalidValue, "OwnerAddr is not set") - } - - request = &Request{ - metadata: *metadata, - } - request.rootAddr, request.metaHash, _, err = request.metadata.serializeAndHash() - request.isNew = true - return request, nil -} - -// Frequency returns the resource's expected update frequency -func (r *Request) Frequency() uint64 { - return r.metadata.Frequency -} - -// Name returns the resource human-readable name -func (r *Request) Name() string { - return r.metadata.Name -} - -// Multihash returns true if the resource data should be interpreted as a multihash -func (r *Request) Multihash() bool { - return r.multihash -} - -// Period returns in which period the resource will be published -func (r *Request) Period() uint32 { - return r.period -} - -// Version returns the resource version to publish -func (r *Request) Version() uint32 { - return r.version -} - -// RootAddr returns the metadata chunk address -func (r *Request) RootAddr() storage.Address { - return r.rootAddr -} - -// StartTime returns the time that the resource was/will be created at -func (r *Request) StartTime() Timestamp { - return r.metadata.StartTime -} - -// Owner returns the resource owner's address -func (r *Request) Owner() common.Address { - return r.metadata.Owner -} - -// Sign executes the signature to validate the resource and sets the owner address field -func (r *Request) Sign(signer Signer) error { - if r.metadata.Owner != zeroAddr && r.metadata.Owner != signer.Address() { - return NewError(ErrInvalidSignature, "Signer does not match current owner of the resource") - } - - if err := r.SignedResourceUpdate.Sign(signer); err != nil { - return err - } - r.metadata.Owner = signer.Address() - return nil -} - -// SetData stores the payload data the resource will be updated with -func (r *Request) SetData(data []byte, multihash bool) { - r.data = data - r.multihash = multihash - r.signature = nil - if !r.isNew { - r.metadata.Frequency = 0 // mark as update - } -} - -func (r *Request) IsNew() bool { - return r.metadata.Frequency > 0 && (r.period <= 1 || r.version <= 1) -} - -func (r *Request) IsUpdate() bool { - return r.signature != nil -} - -// fromJSON takes an update request JSON and populates an UpdateRequest -func (r *Request) fromJSON(j *updateRequestJSON) error { - - r.version = j.Version - r.period = j.Period - r.multihash = j.Multihash - r.metadata.Name = j.Name - r.metadata.Frequency = j.Frequency - r.metadata.StartTime.Time = j.StartTime - - if err := decodeHexArray(r.metadata.Owner[:], j.Owner, "ownerAddr"); err != nil { - return err - } - - var err error - if j.Data != "" { - r.data, err = hexutil.Decode(j.Data) - if err != nil { - return NewError(ErrInvalidValue, "Cannot decode data") - } - } - - var declaredRootAddr storage.Address - var declaredMetaHash []byte - - declaredRootAddr, err = decodeHexSlice(j.RootAddr, storage.AddressLength, "rootAddr") - if err != nil { - return err - } - declaredMetaHash, err = decodeHexSlice(j.MetaHash, 32, "metaHash") - if err != nil { - return err - } - - if r.IsNew() { - // for new resource creation, rootAddr and metaHash are optional because - // we can derive them from the content itself. - // however, if the user sent them, we check them for consistency. - - r.rootAddr, r.metaHash, _, err = r.metadata.serializeAndHash() - if err != nil { - return err - } - if j.RootAddr != "" && !bytes.Equal(declaredRootAddr, r.rootAddr) { - return NewError(ErrInvalidValue, "rootAddr does not match resource metadata") - } - if j.MetaHash != "" && !bytes.Equal(declaredMetaHash, r.metaHash) { - return NewError(ErrInvalidValue, "metaHash does not match resource metadata") - } - - } else { - //Update message - r.rootAddr = declaredRootAddr - r.metaHash = declaredMetaHash - } - - if j.Signature != "" { - sigBytes, err := hexutil.Decode(j.Signature) - if err != nil || len(sigBytes) != signatureLength { - return NewError(ErrInvalidSignature, "Cannot decode signature") - } - r.signature = new(Signature) - r.updateAddr = r.UpdateAddr() - copy(r.signature[:], sigBytes) - } - return nil -} - -func decodeHexArray(dst []byte, src, name string) error { - bytes, err := decodeHexSlice(src, len(dst), name) - if err != nil { - return err - } - if bytes != nil { - copy(dst, bytes) - } - return nil -} - -func decodeHexSlice(src string, expectedLength int, name string) (bytes []byte, err error) { - if src != "" { - bytes, err = hexutil.Decode(src) - if err != nil || len(bytes) != expectedLength { - return nil, NewErrorf(ErrInvalidValue, "Cannot decode %s", name) - } - } - return bytes, nil -} - -// UnmarshalJSON takes a JSON structure stored in a byte array and populates the Request object -// Implements json.Unmarshaler interface -func (r *Request) UnmarshalJSON(rawData []byte) error { - var requestJSON updateRequestJSON - if err := json.Unmarshal(rawData, &requestJSON); err != nil { - return err - } - return r.fromJSON(&requestJSON) -} - -// MarshalJSON takes an update request and encodes it as a JSON structure into a byte array -// Implements json.Marshaler interface -func (r *Request) MarshalJSON() (rawData []byte, err error) { - var signatureString, dataHashString, rootAddrString, metaHashString string - if r.signature != nil { - signatureString = hexutil.Encode(r.signature[:]) - } - if r.data != nil { - dataHashString = hexutil.Encode(r.data) - } - if r.rootAddr != nil { - rootAddrString = hexutil.Encode(r.rootAddr) - } - if r.metaHash != nil { - metaHashString = hexutil.Encode(r.metaHash) - } - var ownerAddrString string - if r.metadata.Frequency == 0 { - ownerAddrString = "" - } else { - ownerAddrString = hexutil.Encode(r.metadata.Owner[:]) - } - - requestJSON := &updateRequestJSON{ - Name: r.metadata.Name, - Frequency: r.metadata.Frequency, - StartTime: r.metadata.StartTime.Time, - Version: r.version, - Period: r.period, - Owner: ownerAddrString, - Data: dataHashString, - Multihash: r.multihash, - Signature: signatureString, - RootAddr: rootAddrString, - MetaHash: metaHashString, - } - - return json.Marshal(requestJSON) -} diff --git a/swarm/storage/mru/request_test.go b/swarm/storage/mru/request_test.go deleted file mode 100644 index dba55b27e3..0000000000 --- a/swarm/storage/mru/request_test.go +++ /dev/null @@ -1,175 +0,0 @@ -package mru - -import ( - "encoding/binary" - "encoding/json" - "fmt" - "reflect" - "testing" -) - -func areEqualJSON(s1, s2 string) (bool, error) { - //credit for the trick: turtlemonvh https://gist.github.com/turtlemonvh/e4f7404e28387fadb8ad275a99596f67 - var o1 interface{} - var o2 interface{} - - err := json.Unmarshal([]byte(s1), &o1) - if err != nil { - return false, fmt.Errorf("Error mashalling string 1 :: %s", err.Error()) - } - err = json.Unmarshal([]byte(s2), &o2) - if err != nil { - return false, fmt.Errorf("Error mashalling string 2 :: %s", err.Error()) - } - - return reflect.DeepEqual(o1, o2), nil -} - -// TestEncodingDecodingUpdateRequests ensures that requests are serialized properly -// while also checking cryptographically that only the owner of a resource can update it. -func TestEncodingDecodingUpdateRequests(t *testing.T) { - - signer := newCharlieSigner() //Charlie, our good guy - falseSigner := newBobSigner() //Bob will play the bad guy again - - // Create a resource to our good guy Charlie's name - createRequest, err := NewCreateRequest(&ResourceMetadata{ - Name: "a good resource name", - Frequency: 300, - StartTime: Timestamp{Time: 1528900000}, - Owner: signer.Address()}) - - if err != nil { - t.Fatalf("Error creating resource name: %s", err) - } - - // We now encode the create message to simulate we send it over the wire - messageRawData, err := createRequest.MarshalJSON() - if err != nil { - t.Fatalf("Error encoding create resource request: %s", err) - } - - // ... the message arrives and is decoded... - var recoveredCreateRequest Request - if err := recoveredCreateRequest.UnmarshalJSON(messageRawData); err != nil { - t.Fatalf("Error decoding create resource request: %s", err) - } - - // ... but verification should fail because it is not signed! - if err := recoveredCreateRequest.Verify(); err == nil { - t.Fatal("Expected Verify to fail since the message is not signed") - } - - // We now assume that the resource was created and propagated. With rootAddr we can retrieve the resource metadata - // and recover the information above. To sign an update, we need the rootAddr and the metaHash to construct - // proof of ownership - - metaHash := createRequest.metaHash - rootAddr := createRequest.rootAddr - const expectedSignature = "0x1c2bab66dc4ed63783d62934e3a628e517888d6949aef0349f3bd677121db9aa09bbfb865904e6c50360e209e0fe6fe757f8a2474cf1b34169c99b95e3fd5a5101" - const expectedJSON = `{"rootAddr":"0x6e744a730f7ea0881528576f0354b6268b98e35a6981ef703153ff1b8d32bbef","metaHash":"0x0c0d5c18b89da503af92302a1a64fab6acb60f78e288eb9c3d541655cd359b60","version":1,"period":7,"data":"0x5468697320686f75722773207570646174653a20537761726d2039392e3020686173206265656e2072656c656173656421","multiHash":false}` - - //Put together an unsigned update request that we will serialize to send it to the signer. - data := []byte("This hour's update: Swarm 99.0 has been released!") - request := &Request{ - SignedResourceUpdate: SignedResourceUpdate{ - resourceUpdate: resourceUpdate{ - updateHeader: updateHeader{ - UpdateLookup: UpdateLookup{ - period: 7, - version: 1, - rootAddr: rootAddr, - }, - multihash: false, - metaHash: metaHash, - }, - data: data, - }, - }, - } - - messageRawData, err = request.MarshalJSON() - if err != nil { - t.Fatalf("Error encoding update request: %s", err) - } - - equalJSON, err := areEqualJSON(string(messageRawData), expectedJSON) - if err != nil { - t.Fatalf("Error decoding update request JSON: %s", err) - } - if !equalJSON { - t.Fatalf("Received a different JSON message. Expected %s, got %s", expectedJSON, string(messageRawData)) - } - - // now the encoded message messageRawData is sent over the wire and arrives to the signer - - //Attempt to extract an UpdateRequest out of the encoded message - var recoveredRequest Request - if err := recoveredRequest.UnmarshalJSON(messageRawData); err != nil { - t.Fatalf("Error decoding update request: %s", err) - } - - //sign the request and see if it matches our predefined signature above. - if err := recoveredRequest.Sign(signer); err != nil { - t.Fatalf("Error signing request: %s", err) - } - - compareByteSliceToExpectedHex(t, "signature", recoveredRequest.signature[:], expectedSignature) - - // mess with the signature and see what happens. To alter the signature, we briefly decode it as JSON - // to alter the signature field. - var j updateRequestJSON - if err := json.Unmarshal([]byte(expectedJSON), &j); err != nil { - t.Fatal("Error unmarshalling test json, check expectedJSON constant") - } - j.Signature = "Certainly not a signature" - corruptMessage, _ := json.Marshal(j) // encode the message with the bad signature - var corruptRequest Request - if err = corruptRequest.UnmarshalJSON(corruptMessage); err == nil { - t.Fatal("Expected DecodeUpdateRequest to fail when trying to interpret a corrupt message with an invalid signature") - } - - // Now imagine Evil Bob (why always Bob, poor Bob) attempts to update Charlie's resource, - // signing a message with his private key - if err := request.Sign(falseSigner); err != nil { - t.Fatalf("Error signing: %s", err) - } - - // Now Bob encodes the message to send it over the wire... - messageRawData, err = request.MarshalJSON() - if err != nil { - t.Fatalf("Error encoding message:%s", err) - } - - // ... the message arrives to our Swarm node and it is decoded. - recoveredRequest = Request{} - if err := recoveredRequest.UnmarshalJSON(messageRawData); err != nil { - t.Fatalf("Error decoding message:%s", err) - } - - // Before discovering Bob's misdemeanor, let's see what would happen if we mess - // with the signature big time to see if Verify catches it - savedSignature := *recoveredRequest.signature // save the signature for later - binary.LittleEndian.PutUint64(recoveredRequest.signature[5:], 556845463424) // write some random data to break the signature - if err = recoveredRequest.Verify(); err == nil { - t.Fatal("Expected Verify to fail on corrupt signature") - } - - // restore the Evil Bob's signature from corruption - *recoveredRequest.signature = savedSignature - - // Now the signature is not corrupt, however Verify should now fail because Bob doesn't own the resource - if err = recoveredRequest.Verify(); err == nil { - t.Fatalf("Expected Verify to fail because this resource belongs to Charlie, not Bob the attacker:%s", err) - } - - // Sign with our friend Charlie's private key - if err := recoveredRequest.Sign(signer); err != nil { - t.Fatalf("Error signing with the correct private key: %s", err) - } - - // And now, Verify should work since this resource belongs to Charlie - if err = recoveredRequest.Verify(); err != nil { - t.Fatalf("Error verifying that Charlie, the good guy, can sign his resource:%s", err) - } -} diff --git a/swarm/storage/mru/resource.go b/swarm/storage/mru/resource.go deleted file mode 100644 index aa83ff62a9..0000000000 --- a/swarm/storage/mru/resource.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "bytes" - "context" - "time" - - "github.com/ethereum/go-ethereum/swarm/storage" -) - -const ( - defaultStoreTimeout = 4000 * time.Millisecond - hasherCount = 8 - resourceHashAlgorithm = storage.SHA3Hash - defaultRetrieveTimeout = 100 * time.Millisecond -) - -// resource caches resource data and the metadata of its root chunk. -type resource struct { - resourceUpdate - ResourceMetadata - *bytes.Reader - lastKey storage.Address - updated time.Time -} - -func (r *resource) Context() context.Context { - return context.TODO() -} - -// TODO Expire content after a defined period (to force resync) -func (r *resource) isSynced() bool { - return !r.updated.IsZero() -} - -// implements storage.LazySectionReader -func (r *resource) Size(ctx context.Context, _ chan bool) (int64, error) { - if !r.isSynced() { - return 0, NewError(ErrNotSynced, "Not synced") - } - return int64(len(r.resourceUpdate.data)), nil -} - -//returns the resource's human-readable name -func (r *resource) Name() string { - return r.ResourceMetadata.Name -} - -// Helper function to calculate the next update period number from the current time, start time and frequency -func getNextPeriod(start uint64, current uint64, frequency uint64) (uint32, error) { - if current < start { - return 0, NewErrorf(ErrInvalidValue, "given current time value %d < start time %d", current, start) - } - if frequency == 0 { - return 0, NewError(ErrInvalidValue, "frequency is 0") - } - timeDiff := current - start - period := timeDiff / frequency - return uint32(period + 1), nil -} diff --git a/swarm/storage/mru/resource_test.go b/swarm/storage/mru/resource_test.go deleted file mode 100644 index 0fb465bb0f..0000000000 --- a/swarm/storage/mru/resource_test.go +++ /dev/null @@ -1,902 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "bytes" - "context" - "crypto/rand" - "encoding/binary" - "flag" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/ethereum/go-ethereum/contracts/ens" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/multihash" - "github.com/ethereum/go-ethereum/swarm/storage" -) - -var ( - loglevel = flag.Int("loglevel", 3, "loglevel") - testHasher = storage.MakeHashFunc(resourceHashAlgorithm)() - startTime = Timestamp{ - Time: uint64(4200), - } - resourceFrequency = uint64(42) - cleanF func() - resourceName = "føø.bar" - hashfunc = storage.MakeHashFunc(storage.DefaultHash) -) - -func init() { - flag.Parse() - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - -// simulated timeProvider -type fakeTimeProvider struct { - currentTime uint64 -} - -func (f *fakeTimeProvider) Tick() { - f.currentTime++ -} - -func (f *fakeTimeProvider) Now() Timestamp { - return Timestamp{ - Time: f.currentTime, - } -} - -func TestUpdateChunkSerializationErrorChecking(t *testing.T) { - - // Test that parseUpdate fails if the chunk is too small - var r SignedResourceUpdate - if err := r.fromChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1)); err == nil { - t.Fatalf("Expected parseUpdate to fail when chunkData contains less than %d bytes", minimumUpdateDataLength) - } - - r = SignedResourceUpdate{} - // Test that parseUpdate fails when the length header does not match the data array length - fakeChunk := make([]byte, 150) - binary.LittleEndian.PutUint16(fakeChunk, 44) - if err := r.fromChunk(storage.ZeroAddr, fakeChunk); err == nil { - t.Fatal("Expected parseUpdate to fail when the header length does not match the actual data array passed in") - } - - r = SignedResourceUpdate{ - resourceUpdate: resourceUpdate{ - updateHeader: updateHeader{ - UpdateLookup: UpdateLookup{ - rootAddr: make([]byte, 79), // put the wrong length, should be storage.AddressLength - }, - metaHash: nil, - multihash: false, - }, - }, - } - _, err := r.toChunk() - if err == nil { - t.Fatal("Expected newUpdateChunk to fail when rootAddr or metaHash have the wrong length") - } - r.rootAddr = make([]byte, storage.AddressLength) - r.metaHash = make([]byte, storage.AddressLength) - _, err = r.toChunk() - if err == nil { - t.Fatal("Expected newUpdateChunk to fail when there is no data") - } - r.data = make([]byte, 79) // put some arbitrary length data - _, err = r.toChunk() - if err == nil { - t.Fatal("expected newUpdateChunk to fail when there is no signature", err) - } - - alice := newAliceSigner() - if err := r.Sign(alice); err != nil { - t.Fatalf("error signing:%s", err) - - } - _, err = r.toChunk() - if err != nil { - t.Fatalf("error creating update chunk:%s", err) - } - - r.multihash = true - r.data[1] = 79 // mess with the multihash, corrupting one byte of it. - if err := r.Sign(alice); err == nil { - t.Fatal("expected Sign() to fail when an invalid multihash is in data and multihash=true", err) - } -} - -// check that signature address matches update signer address -func TestReverse(t *testing.T) { - - period := uint32(4) - version := uint32(2) - - // make fake timeProvider - timeProvider := &fakeTimeProvider{ - currentTime: startTime.Time, - } - - // signer containing private key - signer := newAliceSigner() - - // set up rpc and create resourcehandler - _, _, teardownTest, err := setupTest(timeProvider, signer) - if err != nil { - t.Fatal(err) - } - defer teardownTest() - - metadata := ResourceMetadata{ - Name: resourceName, - StartTime: startTime, - Frequency: resourceFrequency, - Owner: signer.Address(), - } - - rootAddr, metaHash, _, err := metadata.serializeAndHash() - if err != nil { - t.Fatal(err) - } - - // generate some bogus data for the chunk and sign it - data := make([]byte, 8) - _, err = rand.Read(data) - if err != nil { - t.Fatal(err) - } - testHasher.Reset() - testHasher.Write(data) - - update := &SignedResourceUpdate{ - resourceUpdate: resourceUpdate{ - updateHeader: updateHeader{ - UpdateLookup: UpdateLookup{ - period: period, - version: version, - rootAddr: rootAddr, - }, - metaHash: metaHash, - }, - data: data, - }, - } - // generate a hash for t=4200 version 1 - key := update.UpdateAddr() - - if err = update.Sign(signer); err != nil { - t.Fatal(err) - } - - chunk, err := update.toChunk() - if err != nil { - t.Fatal(err) - } - - // check that we can recover the owner account from the update chunk's signature - var checkUpdate SignedResourceUpdate - if err := checkUpdate.fromChunk(chunk.Address(), chunk.Data()); err != nil { - t.Fatal(err) - } - checkdigest, err := checkUpdate.GetDigest() - if err != nil { - t.Fatal(err) - } - recoveredaddress, err := getOwner(checkdigest, *checkUpdate.signature) - if err != nil { - t.Fatalf("Retrieve address from signature fail: %v", err) - } - originaladdress := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) - - // check that the metadata retrieved from the chunk matches what we gave it - if recoveredaddress != originaladdress { - t.Fatalf("addresses dont match: %x != %x", originaladdress, recoveredaddress) - } - - if !bytes.Equal(key[:], chunk.Address()[:]) { - t.Fatalf("Expected chunk key '%x', was '%x'", key, chunk.Address()) - } - if period != checkUpdate.period { - t.Fatalf("Expected period '%d', was '%d'", period, checkUpdate.period) - } - if version != checkUpdate.version { - t.Fatalf("Expected version '%d', was '%d'", version, checkUpdate.version) - } - if !bytes.Equal(data, checkUpdate.data) { - t.Fatalf("Expectedn data '%x', was '%x'", data, checkUpdate.data) - } -} - -// make updates and retrieve them based on periods and versions -func TestResourceHandler(t *testing.T) { - - // make fake timeProvider - timeProvider := &fakeTimeProvider{ - currentTime: startTime.Time, - } - - // signer containing private key - signer := newAliceSigner() - - rh, datadir, teardownTest, err := setupTest(timeProvider, signer) - if err != nil { - t.Fatal(err) - } - defer teardownTest() - - // create a new resource - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - metadata := &ResourceMetadata{ - Name: resourceName, - Frequency: resourceFrequency, - StartTime: Timestamp{Time: timeProvider.Now().Time}, - Owner: signer.Address(), - } - - request, err := NewCreateUpdateRequest(metadata) - if err != nil { - t.Fatal(err) - } - request.Sign(signer) - if err != nil { - t.Fatal(err) - } - err = rh.New(ctx, request) - if err != nil { - t.Fatal(err) - } - - chunk, err := rh.chunkStore.Get(ctx, storage.Address(request.rootAddr)) - if err != nil { - t.Fatal(err) - } else if len(chunk.Data()) < 16 { - t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.Data())) - } - - var recoveredMetadata ResourceMetadata - - recoveredMetadata.binaryGet(chunk.Data()) - if err != nil { - t.Fatal(err) - } - if recoveredMetadata.StartTime.Time != timeProvider.currentTime { - t.Fatalf("stored startTime %d does not match provided startTime %d", recoveredMetadata.StartTime.Time, timeProvider.currentTime) - } - if recoveredMetadata.Frequency != resourceFrequency { - t.Fatalf("stored frequency %d does not match provided frequency %d", recoveredMetadata.Frequency, resourceFrequency) - } - - // data for updates: - updates := []string{ - "blinky", - "pinky", - "inky", - "clyde", - } - - // update halfway to first period. period=1, version=1 - resourcekey := make(map[string]storage.Address) - fwdClock(int(resourceFrequency/2), timeProvider) - data := []byte(updates[0]) - request.SetData(data, false) - if err := request.Sign(signer); err != nil { - t.Fatal(err) - } - resourcekey[updates[0]], err = rh.Update(ctx, &request.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - // update on first period with version = 1 to make it fail since there is already one update with version=1 - request, err = rh.NewUpdateRequest(ctx, request.rootAddr) - if err != nil { - t.Fatal(err) - } - if request.version != 2 || request.period != 1 { - t.Fatal("Suggested period should be 1 and version should be 2") - } - - request.version = 1 // force version 1 instead of 2 to make it fail - data = []byte(updates[1]) - request.SetData(data, false) - if err := request.Sign(signer); err != nil { - t.Fatal(err) - } - resourcekey[updates[1]], err = rh.Update(ctx, &request.SignedResourceUpdate) - if err == nil { - t.Fatal("Expected update to fail since this version already exists") - } - - // update on second period with version = 1, correct. period=2, version=1 - fwdClock(int(resourceFrequency/2), timeProvider) - request, err = rh.NewUpdateRequest(ctx, request.rootAddr) - if err != nil { - t.Fatal(err) - } - request.SetData(data, false) - if err := request.Sign(signer); err != nil { - t.Fatal(err) - } - resourcekey[updates[1]], err = rh.Update(ctx, &request.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - fwdClock(int(resourceFrequency), timeProvider) - // Update on third period, with version = 1 - request, err = rh.NewUpdateRequest(ctx, request.rootAddr) - if err != nil { - t.Fatal(err) - } - data = []byte(updates[2]) - request.SetData(data, false) - if err := request.Sign(signer); err != nil { - t.Fatal(err) - } - resourcekey[updates[2]], err = rh.Update(ctx, &request.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - // update just after third period - fwdClock(1, timeProvider) - request, err = rh.NewUpdateRequest(ctx, request.rootAddr) - if err != nil { - t.Fatal(err) - } - if request.period != 3 || request.version != 2 { - t.Fatal("Suggested period should be 3 and version should be 2") - } - data = []byte(updates[3]) - request.SetData(data, false) - - if err := request.Sign(signer); err != nil { - t.Fatal(err) - } - resourcekey[updates[3]], err = rh.Update(ctx, &request.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - time.Sleep(time.Second) - rh.Close() - - // check we can retrieve the updates after close - // it will match on second iteration startTime + (resourceFrequency * 3) - fwdClock(int(resourceFrequency*2)-1, timeProvider) - - rhparams := &HandlerParams{} - - rh2, err := NewTestHandler(datadir, rhparams) - if err != nil { - t.Fatal(err) - } - - rsrc2, err := rh2.Load(context.TODO(), request.rootAddr) - if err != nil { - t.Fatal(err) - } - - _, err = rh2.Lookup(ctx, LookupLatest(request.rootAddr)) - if err != nil { - t.Fatal(err) - } - - // last update should be "clyde", version two, time= startTime + (resourcefrequency * 3) - if !bytes.Equal(rsrc2.data, []byte(updates[len(updates)-1])) { - t.Fatalf("resource data was %v, expected %v", string(rsrc2.data), updates[len(updates)-1]) - } - if rsrc2.version != 2 { - t.Fatalf("resource version was %d, expected 2", rsrc2.version) - } - if rsrc2.period != 3 { - t.Fatalf("resource period was %d, expected 3", rsrc2.period) - } - log.Debug("Latest lookup", "period", rsrc2.period, "version", rsrc2.version, "data", rsrc2.data) - - // specific period, latest version - rsrc, err := rh2.Lookup(ctx, LookupLatestVersionInPeriod(request.rootAddr, 3)) - if err != nil { - t.Fatal(err) - } - // check data - if !bytes.Equal(rsrc.data, []byte(updates[len(updates)-1])) { - t.Fatalf("resource data (historical) was %v, expected %v", string(rsrc2.data), updates[len(updates)-1]) - } - log.Debug("Historical lookup", "period", rsrc2.period, "version", rsrc2.version, "data", rsrc2.data) - - // specific period, specific version - lookupParams := LookupVersion(request.rootAddr, 3, 1) - rsrc, err = rh2.Lookup(ctx, lookupParams) - if err != nil { - t.Fatal(err) - } - // check data - if !bytes.Equal(rsrc.data, []byte(updates[2])) { - t.Fatalf("resource data (historical) was %v, expected %v", string(rsrc2.data), updates[2]) - } - log.Debug("Specific version lookup", "period", rsrc2.period, "version", rsrc2.version, "data", rsrc2.data) - - // we are now at third update - // check backwards stepping to the first - for i := 1; i >= 0; i-- { - rsrc, err := rh2.LookupPrevious(ctx, lookupParams) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(rsrc.data, []byte(updates[i])) { - t.Fatalf("resource data (previous) was %v, expected %v", rsrc.data, updates[i]) - - } - } - - // beyond the first should yield an error - rsrc, err = rh2.LookupPrevious(ctx, lookupParams) - if err == nil { - t.Fatalf("expected previous to fail, returned period %d version %d data %v", rsrc.period, rsrc.version, rsrc.data) - } - -} - -func TestMultihash(t *testing.T) { - - // make fake timeProvider - timeProvider := &fakeTimeProvider{ - currentTime: startTime.Time, - } - - // signer containing private key - signer := newAliceSigner() - - // set up rpc and create resourcehandler - rh, datadir, teardownTest, err := setupTest(timeProvider, signer) - if err != nil { - t.Fatal(err) - } - defer teardownTest() - - // create a new resource - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - metadata := &ResourceMetadata{ - Name: resourceName, - Frequency: resourceFrequency, - StartTime: Timestamp{Time: timeProvider.Now().Time}, - Owner: signer.Address(), - } - - mr, err := NewCreateRequest(metadata) - if err != nil { - t.Fatal(err) - } - err = rh.New(ctx, mr) - if err != nil { - t.Fatal(err) - } - - // we're naïvely assuming keccak256 for swarm hashes - // if it ever changes this test should also change - multihashbytes := ens.EnsNode("foo") - multihashmulti := multihash.ToMultihash(multihashbytes.Bytes()) - if err != nil { - t.Fatal(err) - } - mr.SetData(multihashmulti, true) - mr.Sign(signer) - if err != nil { - t.Fatal(err) - } - multihashkey, err := rh.Update(ctx, &mr.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - sha1bytes := make([]byte, multihash.MultihashLength) - sha1multi := multihash.ToMultihash(sha1bytes) - if err != nil { - t.Fatal(err) - } - mr, err = rh.NewUpdateRequest(ctx, mr.rootAddr) - if err != nil { - t.Fatal(err) - } - mr.SetData(sha1multi, true) - mr.Sign(signer) - if err != nil { - t.Fatal(err) - } - sha1key, err := rh.Update(ctx, &mr.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - // invalid multihashes - mr, err = rh.NewUpdateRequest(ctx, mr.rootAddr) - if err != nil { - t.Fatal(err) - } - mr.SetData(multihashmulti[1:], true) - mr.Sign(signer) - if err != nil { - t.Fatal(err) - } - _, err = rh.Update(ctx, &mr.SignedResourceUpdate) - if err == nil { - t.Fatalf("Expected update to fail with first byte skipped") - } - mr, err = rh.NewUpdateRequest(ctx, mr.rootAddr) - if err != nil { - t.Fatal(err) - } - mr.SetData(multihashmulti[:len(multihashmulti)-2], true) - mr.Sign(signer) - if err != nil { - t.Fatal(err) - } - - _, err = rh.Update(ctx, &mr.SignedResourceUpdate) - if err == nil { - t.Fatalf("Expected update to fail with last byte skipped") - } - - data, err := getUpdateDirect(rh.Handler, multihashkey) - if err != nil { - t.Fatal(err) - } - multihashdecode, err := multihash.FromMultihash(data) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(multihashdecode, multihashbytes.Bytes()) { - t.Fatalf("Decoded hash '%x' does not match original hash '%x'", multihashdecode, multihashbytes.Bytes()) - } - data, err = getUpdateDirect(rh.Handler, sha1key) - if err != nil { - t.Fatal(err) - } - shadecode, err := multihash.FromMultihash(data) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(shadecode, sha1bytes) { - t.Fatalf("Decoded hash '%x' does not match original hash '%x'", shadecode, sha1bytes) - } - rh.Close() - - rhparams := &HandlerParams{} - // test with signed data - rh2, err := NewTestHandler(datadir, rhparams) - if err != nil { - t.Fatal(err) - } - mr, err = NewCreateRequest(metadata) - if err != nil { - t.Fatal(err) - } - err = rh2.New(ctx, mr) - if err != nil { - t.Fatal(err) - } - - mr.SetData(multihashmulti, true) - mr.Sign(signer) - - if err != nil { - t.Fatal(err) - } - multihashsignedkey, err := rh2.Update(ctx, &mr.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - mr, err = rh2.NewUpdateRequest(ctx, mr.rootAddr) - if err != nil { - t.Fatal(err) - } - mr.SetData(sha1multi, true) - mr.Sign(signer) - if err != nil { - t.Fatal(err) - } - - sha1signedkey, err := rh2.Update(ctx, &mr.SignedResourceUpdate) - if err != nil { - t.Fatal(err) - } - - data, err = getUpdateDirect(rh2.Handler, multihashsignedkey) - if err != nil { - t.Fatal(err) - } - multihashdecode, err = multihash.FromMultihash(data) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(multihashdecode, multihashbytes.Bytes()) { - t.Fatalf("Decoded hash '%x' does not match original hash '%x'", multihashdecode, multihashbytes.Bytes()) - } - data, err = getUpdateDirect(rh2.Handler, sha1signedkey) - if err != nil { - t.Fatal(err) - } - shadecode, err = multihash.FromMultihash(data) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(shadecode, sha1bytes) { - t.Fatalf("Decoded hash '%x' does not match original hash '%x'", shadecode, sha1bytes) - } -} - -// \TODO verify testing of signature validation and enforcement -func TestValidator(t *testing.T) { - - // make fake timeProvider - timeProvider := &fakeTimeProvider{ - currentTime: startTime.Time, - } - - // signer containing private key. Alice will be the good girl - signer := newAliceSigner() - - // fake signer for false results. Bob will play the bad guy today. - falseSigner := newBobSigner() - - // set up sim timeProvider - rh, _, teardownTest, err := setupTest(timeProvider, signer) - if err != nil { - t.Fatal(err) - } - defer teardownTest() - - // create new resource - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - metadata := &ResourceMetadata{ - Name: resourceName, - Frequency: resourceFrequency, - StartTime: Timestamp{Time: timeProvider.Now().Time}, - Owner: signer.Address(), - } - mr, err := NewCreateRequest(metadata) - if err != nil { - t.Fatal(err) - } - mr.Sign(signer) - - err = rh.New(ctx, mr) - if err != nil { - t.Fatalf("Create resource fail: %v", err) - } - - // chunk with address - data := []byte("foo") - mr.SetData(data, false) - if err := mr.Sign(signer); err != nil { - t.Fatalf("sign fail: %v", err) - } - chunk, err := mr.SignedResourceUpdate.toChunk() - if err != nil { - t.Fatal(err) - } - if !rh.Validate(chunk.Address(), chunk.Data()) { - t.Fatal("Chunk validator fail on update chunk") - } - - // chunk with address made from different publickey - if err := mr.Sign(falseSigner); err == nil { - t.Fatalf("Expected Sign to fail since we are using a different OwnerAddr: %v", err) - } - - // chunk with address made from different publickey - mr.metadata.Owner = zeroAddr // set to zero to bypass .Sign() check - if err := mr.Sign(falseSigner); err != nil { - t.Fatalf("sign fail: %v", err) - } - - chunk, err = mr.SignedResourceUpdate.toChunk() - if err != nil { - t.Fatal(err) - } - - if rh.Validate(chunk.Address(), chunk.Data()) { - t.Fatal("Chunk validator did not fail on update chunk with false address") - } - - ctx, cancel = context.WithTimeout(context.Background(), time.Second) - defer cancel() - - metadata = &ResourceMetadata{ - Name: resourceName, - StartTime: TimestampProvider.Now(), - Frequency: resourceFrequency, - Owner: signer.Address(), - } - chunk, _, err = metadata.newChunk() - if err != nil { - t.Fatal(err) - } - - if !rh.Validate(chunk.Address(), chunk.Data()) { - t.Fatal("Chunk validator fail on metadata chunk") - } -} - -// tests that the content address validator correctly checks the data -// tests that resource update chunks are passed through content address validator -// there is some redundancy in this test as it also tests content addressed chunks, -// which should be evaluated as invalid chunks by this validator -func TestValidatorInStore(t *testing.T) { - - // make fake timeProvider - TimestampProvider = &fakeTimeProvider{ - currentTime: startTime.Time, - } - - // signer containing private key - signer := newAliceSigner() - - // set up localstore - datadir, err := ioutil.TempDir("", "storage-testresourcevalidator") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(datadir) - - params := storage.NewDefaultLocalStoreParams() - params.Init(datadir) - store, err := storage.NewLocalStore(params, nil) - if err != nil { - t.Fatal(err) - } - - // set up resource handler and add is as a validator to the localstore - rhParams := &HandlerParams{} - rh := NewHandler(rhParams) - store.Validators = append(store.Validators, rh) - - // create content addressed chunks, one good, one faulty - chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) - goodChunk := chunks[0] - badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data()) - - metadata := &ResourceMetadata{ - StartTime: startTime, - Name: "xyzzy", - Frequency: resourceFrequency, - Owner: signer.Address(), - } - - rootChunk, metaHash, err := metadata.newChunk() - if err != nil { - t.Fatal(err) - } - // create a resource update chunk with correct publickey - updateLookup := UpdateLookup{ - period: 42, - version: 1, - rootAddr: rootChunk.Address(), - } - - updateAddr := updateLookup.UpdateAddr() - data := []byte("bar") - - r := SignedResourceUpdate{ - updateAddr: updateAddr, - resourceUpdate: resourceUpdate{ - updateHeader: updateHeader{ - UpdateLookup: updateLookup, - metaHash: metaHash, - }, - data: data, - }, - } - - r.Sign(signer) - - uglyChunk, err := r.toChunk() - if err != nil { - t.Fatal(err) - } - - // put the chunks in the store and check their error status - err = store.Put(context.Background(), goodChunk) - if err == nil { - t.Fatal("expected error on good content address chunk with resource validator only, but got nil") - } - err = store.Put(context.Background(), badChunk) - if err == nil { - t.Fatal("expected error on bad content address chunk with resource validator only, but got nil") - } - err = store.Put(context.Background(), uglyChunk) - if err != nil { - t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err) - } -} - -// fast-forward clock -func fwdClock(count int, timeProvider *fakeTimeProvider) { - for i := 0; i < count; i++ { - timeProvider.Tick() - } -} - -// create rpc and resourcehandler -func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler, datadir string, teardown func(), err error) { - - var fsClean func() - var rpcClean func() - cleanF = func() { - if fsClean != nil { - fsClean() - } - if rpcClean != nil { - rpcClean() - } - } - - // temp datadir - datadir, err = ioutil.TempDir("", "rh") - if err != nil { - return nil, "", nil, err - } - fsClean = func() { - os.RemoveAll(datadir) - } - - TimestampProvider = timeProvider - rhparams := &HandlerParams{} - rh, err = NewTestHandler(datadir, rhparams) - return rh, datadir, cleanF, err -} - -func newAliceSigner() *GenericSigner { - privKey, _ := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") - return NewGenericSigner(privKey) -} - -func newBobSigner() *GenericSigner { - privKey, _ := crypto.HexToECDSA("accedeaccedeaccedeaccedeaccedeaccedeaccedeaccedeaccedeaccedecaca") - return NewGenericSigner(privKey) -} - -func newCharlieSigner() *GenericSigner { - privKey, _ := crypto.HexToECDSA("facadefacadefacadefacadefacadefacadefacadefacadefacadefacadefaca") - return NewGenericSigner(privKey) -} - -func getUpdateDirect(rh *Handler, addr storage.Address) ([]byte, error) { - chunk, err := rh.chunkStore.Get(context.TODO(), addr) - if err != nil { - return nil, err - } - var r SignedResourceUpdate - if err := r.fromChunk(addr, chunk.Data()); err != nil { - return nil, err - } - return r.data, nil -} diff --git a/swarm/storage/mru/signedupdate.go b/swarm/storage/mru/signedupdate.go deleted file mode 100644 index 41a5a5e631..0000000000 --- a/swarm/storage/mru/signedupdate.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "bytes" - "hash" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/swarm/storage" -) - -// SignedResourceUpdate represents a resource update with all the necessary information to prove ownership of the resource -type SignedResourceUpdate struct { - resourceUpdate // actual content that will be put on the chunk, less signature - signature *Signature - updateAddr storage.Address // resulting chunk address for the update (not serialized, for internal use) - binaryData []byte // resulting serialized data (not serialized, for efficiency/internal use) -} - -// Verify checks that signatures are valid and that the signer owns the resource to be updated -func (r *SignedResourceUpdate) Verify() (err error) { - if len(r.data) == 0 { - return NewError(ErrInvalidValue, "Update does not contain data") - } - if r.signature == nil { - return NewError(ErrInvalidSignature, "Missing signature field") - } - - digest, err := r.GetDigest() - if err != nil { - return err - } - - // get the address of the signer (which also checks that it's a valid signature) - ownerAddr, err := getOwner(digest, *r.signature) - if err != nil { - return err - } - - if !bytes.Equal(r.updateAddr, r.UpdateAddr()) { - return NewError(ErrInvalidSignature, "Signature address does not match with ownerAddr") - } - - // Check if who signed the resource update really owns the resource - if !verifyOwner(ownerAddr, r.metaHash, r.rootAddr) { - return NewErrorf(ErrUnauthorized, "signature is valid but signer does not own the resource: %v", err) - } - - return nil -} - -// Sign executes the signature to validate the resource -func (r *SignedResourceUpdate) Sign(signer Signer) error { - - r.binaryData = nil //invalidate serialized data - digest, err := r.GetDigest() // computes digest and serializes into .binaryData - if err != nil { - return err - } - - signature, err := signer.Sign(digest) - if err != nil { - return err - } - - // Although the Signer interface returns the public address of the signer, - // recover it from the signature to see if they match - ownerAddress, err := getOwner(digest, signature) - if err != nil { - return NewError(ErrInvalidSignature, "Error verifying signature") - } - - if ownerAddress != signer.Address() { // sanity check to make sure the Signer is declaring the same address used to sign! - return NewError(ErrInvalidSignature, "Signer address does not match ownerAddr") - } - - r.signature = &signature - r.updateAddr = r.UpdateAddr() - return nil -} - -// create an update chunk. -func (r *SignedResourceUpdate) toChunk() (storage.Chunk, error) { - - // Check that the update is signed and serialized - // For efficiency, data is serialized during signature and cached in - // the binaryData field when computing the signature digest in .getDigest() - if r.signature == nil || r.binaryData == nil { - return nil, NewError(ErrInvalidSignature, "newUpdateChunk called without a valid signature or payload data. Call .Sign() first.") - } - - resourceUpdateLength := r.resourceUpdate.binaryLength() - // signature is the last item in the chunk data - copy(r.binaryData[resourceUpdateLength:], r.signature[:]) - - chunk := storage.NewChunk(r.updateAddr, r.binaryData) - return chunk, nil -} - -// fromChunk populates this structure from chunk data. It does not verify the signature is valid. -func (r *SignedResourceUpdate) fromChunk(updateAddr storage.Address, chunkdata []byte) error { - // for update chunk layout see SignedResourceUpdate definition - - //deserialize the resource update portion - if err := r.resourceUpdate.binaryGet(chunkdata); err != nil { - return err - } - - // Extract the signature - var signature *Signature - cursor := r.resourceUpdate.binaryLength() - sigdata := chunkdata[cursor : cursor+signatureLength] - if len(sigdata) > 0 { - signature = &Signature{} - copy(signature[:], sigdata) - } - - r.signature = signature - r.updateAddr = updateAddr - r.binaryData = chunkdata - - return nil - -} - -// GetDigest creates the resource update digest used in signatures (formerly known as keyDataHash) -// the serialized payload is cached in .binaryData -func (r *SignedResourceUpdate) GetDigest() (result common.Hash, err error) { - hasher := hashPool.Get().(hash.Hash) - defer hashPool.Put(hasher) - hasher.Reset() - dataLength := r.resourceUpdate.binaryLength() - if r.binaryData == nil { - r.binaryData = make([]byte, dataLength+signatureLength) - if err := r.resourceUpdate.binaryPut(r.binaryData[:dataLength]); err != nil { - return result, err - } - } - hasher.Write(r.binaryData[:dataLength]) //everything except the signature. - - return common.BytesToHash(hasher.Sum(nil)), nil -} - -// getOwner extracts the address of the resource update signer -func getOwner(digest common.Hash, signature Signature) (common.Address, error) { - pub, err := crypto.SigToPub(digest.Bytes(), signature[:]) - if err != nil { - return common.Address{}, err - } - return crypto.PubkeyToAddress(*pub), nil -} - -// verifyResourceOwnerhsip checks that the signer of the update actually owns the resource -// H(ownerAddr, metaHash) is computed. If it matches the rootAddr the update chunk is claiming -// to update, it is proven that signer of the resource update owns the resource. -// See metadataHash in metadata.go for a more detailed explanation -func verifyOwner(ownerAddr common.Address, metaHash []byte, rootAddr storage.Address) bool { - hasher := hashPool.Get().(hash.Hash) - defer hashPool.Put(hasher) - hasher.Reset() - hasher.Write(metaHash) - hasher.Write(ownerAddr.Bytes()) - rootAddr2 := hasher.Sum(nil) - return bytes.Equal(rootAddr2, rootAddr) -} diff --git a/swarm/storage/mru/update.go b/swarm/storage/mru/update.go deleted file mode 100644 index d1bd37ddff..0000000000 --- a/swarm/storage/mru/update.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "encoding/binary" - "errors" - - "github.com/ethereum/go-ethereum/swarm/chunk" - "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/multihash" -) - -// resourceUpdate encapsulates the information sent as part of a resource update -type resourceUpdate struct { - updateHeader // metainformationa about this resource update - data []byte // actual data payload -} - -// Update chunk layout -// Prefix: -// 2 bytes updateHeaderLength -// 2 bytes data length -const chunkPrefixLength = 2 + 2 - -// Header: (see updateHeader) -// Data: -// data (datalength bytes) -// -// Minimum size is Header + 1 (minimum data length, enforced) -const minimumUpdateDataLength = updateHeaderLength + 1 -const maxUpdateDataLength = chunk.DefaultSize - signatureLength - updateHeaderLength - chunkPrefixLength - -// binaryPut serializes the resource update information into the given slice -func (r *resourceUpdate) binaryPut(serializedData []byte) error { - datalength := len(r.data) - if datalength == 0 { - return NewError(ErrInvalidValue, "cannot update a resource with no data") - } - - if datalength > maxUpdateDataLength { - return NewErrorf(ErrInvalidValue, "data is too big (length=%d). Max length=%d", datalength, maxUpdateDataLength) - } - - if len(serializedData) != r.binaryLength() { - return NewErrorf(ErrInvalidValue, "slice passed to putBinary must be of exact size. Expected %d bytes", r.binaryLength()) - } - - if r.multihash { - if _, _, err := multihash.GetMultihashLength(r.data); err != nil { - return NewError(ErrInvalidValue, "Invalid multihash") - } - } - - // Add prefix: updateHeaderLength and actual data length - cursor := 0 - binary.LittleEndian.PutUint16(serializedData[cursor:], uint16(updateHeaderLength)) - cursor += 2 - - // data length - binary.LittleEndian.PutUint16(serializedData[cursor:], uint16(datalength)) - cursor += 2 - - // serialize header (see updateHeader) - if err := r.updateHeader.binaryPut(serializedData[cursor : cursor+updateHeaderLength]); err != nil { - return err - } - cursor += updateHeaderLength - - // add the data - copy(serializedData[cursor:], r.data) - cursor += datalength - - return nil -} - -// binaryLength returns the expected number of bytes this structure will take to encode -func (r *resourceUpdate) binaryLength() int { - return chunkPrefixLength + updateHeaderLength + len(r.data) -} - -// binaryGet populates this instance from the information contained in the passed byte slice -func (r *resourceUpdate) binaryGet(serializedData []byte) error { - if len(serializedData) < minimumUpdateDataLength { - return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a resource update chunk", minimumUpdateDataLength) - } - cursor := 0 - declaredHeaderlength := binary.LittleEndian.Uint16(serializedData[cursor : cursor+2]) - if declaredHeaderlength != updateHeaderLength { - return NewErrorf(ErrCorruptData, "Invalid header length. Expected %d, got %d", updateHeaderLength, declaredHeaderlength) - } - - cursor += 2 - datalength := int(binary.LittleEndian.Uint16(serializedData[cursor : cursor+2])) - cursor += 2 - - if chunkPrefixLength+updateHeaderLength+datalength+signatureLength != len(serializedData) { - return NewError(ErrNothingToReturn, "length specified in header is different than actual chunk size") - } - - // at this point we can be satisfied that we have the correct data length to read - if err := r.updateHeader.binaryGet(serializedData[cursor : cursor+updateHeaderLength]); err != nil { - return err - } - cursor += updateHeaderLength - - data := serializedData[cursor : cursor+datalength] - cursor += datalength - - // if multihash content is indicated we check the validity of the multihash - if r.updateHeader.multihash { - mhLength, mhHeaderLength, err := multihash.GetMultihashLength(data) - if err != nil { - log.Error("multihash parse error", "err", err) - return err - } - if datalength != mhLength+mhHeaderLength { - log.Debug("multihash error", "datalength", datalength, "mhLength", mhLength, "mhHeaderLength", mhHeaderLength) - return errors.New("Corrupt multihash data") - } - } - - // now that all checks have passed, copy data into structure - r.data = make([]byte, datalength) - copy(r.data, data) - - return nil - -} - -// Multihash specifies whether the resource data should be interpreted as multihash -func (r *resourceUpdate) Multihash() bool { - return r.multihash -} diff --git a/swarm/storage/mru/update_test.go b/swarm/storage/mru/update_test.go deleted file mode 100644 index 51e9d2fcc5..0000000000 --- a/swarm/storage/mru/update_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package mru - -import ( - "bytes" - "testing" -) - -const serializedUpdateHex = "0x490034004f000000da070000fb0ed7efa696bdb0b54cd75554cc3117ffc891454317df7dd6fefad978e2f2fbf74a10ce8f26ffc8bfaa07c3031a34b2c61f517955e7deb1592daccf96c69cf000456c20717565206c6565206d7563686f207920616e6461206d7563686f2c207665206d7563686f20792073616265206d7563686f" -const serializedUpdateMultihashHex = "0x490022004f000000da070000fb0ed7efa696bdb0b54cd75554cc3117ffc891454317df7dd6fefad978e2f2fbf74a10ce8f26ffc8bfaa07c3031a34b2c61f517955e7deb1592daccf96c69cf0011b200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1c1e1f20" - -func getTestResourceUpdate() *resourceUpdate { - return &resourceUpdate{ - updateHeader: *getTestUpdateHeader(false), - data: []byte("El que lee mucho y anda mucho, ve mucho y sabe mucho"), - } -} - -func getTestResourceUpdateMultihash() *resourceUpdate { - return &resourceUpdate{ - updateHeader: *getTestUpdateHeader(true), - data: []byte{0x1b, 0x20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 28, 30, 31, 32}, - } -} - -func compareResourceUpdate(a, b *resourceUpdate) bool { - return compareUpdateHeader(&a.updateHeader, &b.updateHeader) && - bytes.Equal(a.data, b.data) -} - -func TestResourceUpdateSerializer(t *testing.T) { - var serializedUpdateLength = len(serializedUpdateHex)/2 - 1 // hack to calculate the byte length out of the hex representation - update := getTestResourceUpdate() - serializedUpdate := make([]byte, serializedUpdateLength) - if err := update.binaryPut(serializedUpdate); err != nil { - t.Fatal(err) - } - compareByteSliceToExpectedHex(t, "serializedUpdate", serializedUpdate, serializedUpdateHex) - - // Test fail if update does not contain data - update.data = nil - if err := update.binaryPut(serializedUpdate); err == nil { - t.Fatal("Expected resourceUpdate.binaryPut to fail since update does not contain data") - } - - // Test fail if update is too big - update.data = make([]byte, 10000) - if err := update.binaryPut(serializedUpdate); err == nil { - t.Fatal("Expected resourceUpdate.binaryPut to fail since update is too big") - } - - // Test fail if passed slice is not of the exact size required for this update - update.data = make([]byte, 1) - if err := update.binaryPut(serializedUpdate); err == nil { - t.Fatal("Expected resourceUpdate.binaryPut to fail since passed slice is not of the appropriate size") - } - - // Test serializing a multihash update - var serializedUpdateMultihashLength = len(serializedUpdateMultihashHex)/2 - 1 // hack to calculate the byte length out of the hex representation - update = getTestResourceUpdateMultihash() - serializedUpdate = make([]byte, serializedUpdateMultihashLength) - if err := update.binaryPut(serializedUpdate); err != nil { - t.Fatal(err) - } - compareByteSliceToExpectedHex(t, "serializedUpdate", serializedUpdate, serializedUpdateMultihashHex) - - // mess with the multihash to test it fails with a wrong multihash error - update.data[1] = 79 - if err := update.binaryPut(serializedUpdate); err == nil { - t.Fatal("Expected resourceUpdate.binaryPut to fail since data contains an invalid multihash") - } - -} diff --git a/swarm/storage/mru/updateheader.go b/swarm/storage/mru/updateheader.go deleted file mode 100644 index f0039eaf66..0000000000 --- a/swarm/storage/mru/updateheader.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mru - -import ( - "github.com/ethereum/go-ethereum/swarm/storage" -) - -// updateHeader models the non-payload components of a Resource Update -type updateHeader struct { - UpdateLookup // UpdateLookup contains the information required to locate this resource (components of the search key used to find it) - multihash bool // Whether the data in this Resource Update should be interpreted as multihash - metaHash []byte // SHA3 hash of the metadata chunk (less ownerAddr). Used to prove ownerhsip of the resource. -} - -const metaHashLength = storage.AddressLength - -// updateLookupLength bytes -// 1 byte flags (multihash bool for now) -// 32 bytes metaHash -const updateHeaderLength = updateLookupLength + 1 + metaHashLength - -// binaryPut serializes the resource header information into the given slice -func (h *updateHeader) binaryPut(serializedData []byte) error { - if len(serializedData) != updateHeaderLength { - return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize updateHeaderLength. Expected %d, got %d", updateHeaderLength, len(serializedData)) - } - if len(h.metaHash) != metaHashLength { - return NewError(ErrInvalidValue, "updateHeader.binaryPut called without metaHash set") - } - if err := h.UpdateLookup.binaryPut(serializedData[:updateLookupLength]); err != nil { - return err - } - cursor := updateLookupLength - copy(serializedData[cursor:], h.metaHash[:metaHashLength]) - cursor += metaHashLength - - var flags byte - if h.multihash { - flags |= 0x01 - } - - serializedData[cursor] = flags - cursor++ - - return nil -} - -// binaryLength returns the expected size of this structure when serialized -func (h *updateHeader) binaryLength() int { - return updateHeaderLength -} - -// binaryGet restores the current updateHeader instance from the information contained in the passed slice -func (h *updateHeader) binaryGet(serializedData []byte) error { - if len(serializedData) != updateHeaderLength { - return NewErrorf(ErrInvalidValue, "Incorrect slice size to read updateHeaderLength. Expected %d, got %d", updateHeaderLength, len(serializedData)) - } - - if err := h.UpdateLookup.binaryGet(serializedData[:updateLookupLength]); err != nil { - return err - } - cursor := updateLookupLength - h.metaHash = make([]byte, metaHashLength) - copy(h.metaHash[:storage.AddressLength], serializedData[cursor:cursor+storage.AddressLength]) - cursor += metaHashLength - - flags := serializedData[cursor] - cursor++ - - h.multihash = flags&0x01 != 0 - - return nil -} diff --git a/swarm/storage/mru/updateheader_test.go b/swarm/storage/mru/updateheader_test.go deleted file mode 100644 index b1f5059892..0000000000 --- a/swarm/storage/mru/updateheader_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package mru - -import ( - "bytes" - "testing" - - "github.com/ethereum/go-ethereum/common/hexutil" -) - -const serializedUpdateHeaderMultihashHex = "0x4f000000da070000fb0ed7efa696bdb0b54cd75554cc3117ffc891454317df7dd6fefad978e2f2fbf74a10ce8f26ffc8bfaa07c3031a34b2c61f517955e7deb1592daccf96c69cf001" - -func getTestUpdateHeader(multihash bool) (header *updateHeader) { - _, metaHash, _, _ := getTestMetadata().serializeAndHash() - return &updateHeader{ - UpdateLookup: *getTestUpdateLookup(), - multihash: multihash, - metaHash: metaHash, - } -} - -func compareUpdateHeader(a, b *updateHeader) bool { - return compareUpdateLookup(&a.UpdateLookup, &b.UpdateLookup) && - a.multihash == b.multihash && - bytes.Equal(a.metaHash, b.metaHash) -} - -func TestUpdateHeaderSerializer(t *testing.T) { - header := getTestUpdateHeader(true) - serializedHeader := make([]byte, updateHeaderLength) - if err := header.binaryPut(serializedHeader); err != nil { - t.Fatal(err) - } - compareByteSliceToExpectedHex(t, "serializedHeader", serializedHeader, serializedUpdateHeaderMultihashHex) - - // trigger incorrect slice length error passing a slice that is 1 byte too big - if err := header.binaryPut(make([]byte, updateHeaderLength+1)); err == nil { - t.Fatal("Expected updateHeader.binaryPut to fail since supplied slice is of incorrect length") - } - - // trigger invalid metaHash error - header.metaHash = nil - if err := header.binaryPut(serializedHeader); err == nil { - t.Fatal("Expected updateHeader.binaryPut to fail metaHash is of incorrect length") - } -} - -func TestUpdateHeaderDeserializer(t *testing.T) { - originalUpdate := getTestUpdateHeader(true) - serializedData, _ := hexutil.Decode(serializedUpdateHeaderMultihashHex) - var retrievedUpdate updateHeader - if err := retrievedUpdate.binaryGet(serializedData); err != nil { - t.Fatal(err) - } - if !compareUpdateHeader(originalUpdate, &retrievedUpdate) { - t.Fatalf("Expected deserialized structure to equal the original") - } - - // mess with source slice to test length checks - serializedData = []byte{1, 2, 3} - if err := retrievedUpdate.binaryGet(serializedData); err == nil { - t.Fatal("Expected retrievedUpdate.binaryGet, since passed slice is too small") - } - -} diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index a3a5522321..16bc48a9a2 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -34,7 +34,7 @@ type ( ) type NetFetcher interface { - Request(ctx context.Context) + Request(ctx context.Context, hopCount uint8) Offer(ctx context.Context, source *enode.ID) } @@ -263,6 +263,9 @@ func (f *fetcher) Fetch(rctx context.Context) (Chunk, error) { // If there is a source in the context then it is an offer, otherwise a request sourceIF := rctx.Value("source") + + hopCount, _ := rctx.Value("hopcount").(uint8) + if sourceIF != nil { var source enode.ID if err := source.UnmarshalText([]byte(sourceIF.(string))); err != nil { @@ -270,7 +273,7 @@ func (f *fetcher) Fetch(rctx context.Context) (Chunk, error) { } f.netFetcher.Offer(rctx, &source) } else { - f.netFetcher.Request(rctx) + f.netFetcher.Request(rctx, hopCount) } // wait until either the chunk is delivered or the context is done diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index b734c117ba..8a09fa5ae0 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -40,6 +40,7 @@ type mockNetFetcher struct { offerCalled bool quit <-chan struct{} ctx context.Context + hopCounts []uint8 } func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { @@ -47,7 +48,7 @@ func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { m.sources = append(m.sources, source) } -func (m *mockNetFetcher) Request(ctx context.Context) { +func (m *mockNetFetcher) Request(ctx context.Context, hopCount uint8) { m.requestCalled = true var peers []Address m.peers.Range(func(key interface{}, _ interface{}) bool { @@ -55,6 +56,7 @@ func (m *mockNetFetcher) Request(ctx context.Context) { return true }) m.peersPerRequest = append(m.peersPerRequest, peers) + m.hopCounts = append(m.hopCounts, hopCount) } type mockNetFetchFuncFactory struct { @@ -412,7 +414,8 @@ func TestNetStoreGetCallsRequest(t *testing.T) { chunk := GenerateRandomChunk(ch.DefaultSize) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + ctx := context.WithValue(context.Background(), "hopcount", uint8(5)) + ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) defer cancel() // We call get for a not available chunk, it will timeout because the chunk is not delivered @@ -426,6 +429,10 @@ func TestNetStoreGetCallsRequest(t *testing.T) { if !fetcher.requestCalled { t.Fatal("Expected NetFetcher.Request to be called") } + + if fetcher.hopCounts[0] != 5 { + t.Fatalf("Expected NetFetcher.Request be called with hopCount 5, got %v", fetcher.hopCounts[0]) + } } // TestNetStoreGetCallsOffer tests if Get created a request on the NetFetcher for an unavailable chunk diff --git a/swarm/storage/schema.go b/swarm/storage/schema.go new file mode 100644 index 0000000000..91847ca0f9 --- /dev/null +++ b/swarm/storage/schema.go @@ -0,0 +1,17 @@ +package storage + +// The DB schema we want to use. The actual/current DB schema might differ +// until migrations are run. +const CurrentDbSchema = DbSchemaHalloween + +// There was a time when we had no schema at all. +const DbSchemaNone = "" + +// "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5 +const DbSchemaPurity = "purity" + +// "halloween" is here because we had a screw in the garbage collector index. +// Because of that we had to rebuild the GC index to get rid of erroneous +// entries and that takes a long time. This schema is used for bookkeeping, +// so rebuild index will run just once. +const DbSchemaHalloween = "halloween" diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 8c70f45848..42557766ef 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -25,7 +25,6 @@ import ( "fmt" "hash" "io" - "io/ioutil" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" @@ -80,6 +79,19 @@ func (a Address) bits(i, j uint) uint { return res } +// Proximity(x, y) returns the proximity order of the MSB distance between x and y +// +// The distance metric MSB(x, y) of two equal length byte sequences x an y is the +// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. +// the binary cast is big endian: most significant bit first (=MSB). +// +// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. +// It is defined as the reverse rank of the integer part of the base 2 +// logarithm of the distance. +// It is calculated by counting the number of common leading zeros in the (MSB) +// binary representation of the x^y. +// +// (0 farthest, 255 closest, 256 self) func Proximity(one, other []byte) (ret int) { b := (MaxPO-1)/8 + 1 if b > len(one) { @@ -231,26 +243,13 @@ func GenerateRandomChunk(dataSize int64) Chunk { } func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { - if dataSize > ch.DefaultSize { - dataSize = ch.DefaultSize - } for i := 0; i < count; i++ { - ch := GenerateRandomChunk(ch.DefaultSize) + ch := GenerateRandomChunk(dataSize) chunks = append(chunks, ch) } return chunks } -func GenerateRandomData(l int) (r io.Reader, slice []byte) { - slice, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(l))) - if err != nil { - panic("rand error") - } - // log.Warn("generate random data", "len", len(slice), "data", common.Bytes2Hex(slice)) - r = io.LimitReader(bytes.NewReader(slice), int64(l)) - return r, slice -} - // Size, Seek, Read, ReadAt type LazySectionReader interface { Context() context.Context diff --git a/swarm/swap/swap.go b/swarm/swap/swap.go new file mode 100644 index 0000000000..137eb141d9 --- /dev/null +++ b/swarm/swap/swap.go @@ -0,0 +1,93 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swap + +import ( + "errors" + "fmt" + "strconv" + "sync" + + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/state" +) + +// SwAP Swarm Accounting Protocol +// a peer to peer micropayment system +// A node maintains an individual balance with every peer +// Only messages which have a price will be accounted for +type Swap struct { + stateStore state.Store //stateStore is needed in order to keep balances across sessions + lock sync.RWMutex //lock the balances + balances map[enode.ID]int64 //map of balances for each peer +} + +// New - swap constructor +func New(stateStore state.Store) (swap *Swap) { + swap = &Swap{ + stateStore: stateStore, + balances: make(map[enode.ID]int64), + } + return +} + +//Swap implements the protocols.Balance interface +//Add is the (sole) accounting function +func (s *Swap) Add(amount int64, peer *protocols.Peer) (err error) { + s.lock.Lock() + defer s.lock.Unlock() + + //load existing balances from the state store + err = s.loadState(peer) + if err != nil && err != state.ErrNotFound { + return + } + //adjust the balance + //if amount is negative, it will decrease, otherwise increase + s.balances[peer.ID()] += amount + //save the new balance to the state store + peerBalance := s.balances[peer.ID()] + err = s.stateStore.Put(peer.ID().String(), &peerBalance) + + log.Debug(fmt.Sprintf("balance for peer %s: %s", peer.ID().String(), strconv.FormatInt(peerBalance, 10))) + return err +} + +//GetPeerBalance returns the balance for a given peer +func (swap *Swap) GetPeerBalance(peer enode.ID) (int64, error) { + swap.lock.RLock() + defer swap.lock.RUnlock() + if p, ok := swap.balances[peer]; ok { + return p, nil + } + return 0, errors.New("Peer not found") +} + +//load balances from the state store (persisted) +func (s *Swap) loadState(peer *protocols.Peer) (err error) { + var peerBalance int64 + peerID := peer.ID() + //only load if the current instance doesn't already have this peer's + //balance in memory + if _, ok := s.balances[peerID]; !ok { + err = s.stateStore.Get(peerID.String(), &peerBalance) + s.balances[peerID] = peerBalance + } + return +} diff --git a/swarm/swap/swap_test.go b/swarm/swap/swap_test.go new file mode 100644 index 0000000000..f2e3ba168a --- /dev/null +++ b/swarm/swap/swap_test.go @@ -0,0 +1,184 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swap + +import ( + "flag" + "fmt" + "io/ioutil" + mrand "math/rand" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/state" + colorable "github.com/mattn/go-colorable" +) + +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + mrand.Seed(time.Now().UnixNano()) + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + +//Test getting a peer's balance +func TestGetPeerBalance(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + //test for correct value + testPeer := newDummyPeer() + swap.balances[testPeer.ID()] = 888 + b, err := swap.GetPeerBalance(testPeer.ID()) + if err != nil { + t.Fatal(err) + } + if b != 888 { + t.Fatalf("Expected peer's balance to be %d, but is %d", 888, b) + } + + //test for inexistent node + id := adapters.RandomNodeConfig().ID + _, err = swap.GetPeerBalance(id) + if err == nil { + t.Fatal("Expected call to fail, but it didn't!") + } + if err.Error() != "Peer not found" { + t.Fatalf("Expected test to fail with %s, but is %s", "Peer not found", err.Error()) + } +} + +//Test that repeated bookings do correct accounting +func TestRepeatedBookings(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + testPeer := newDummyPeer() + amount := mrand.Intn(100) + cnt := 1 + mrand.Intn(10) + for i := 0; i < cnt; i++ { + swap.Add(int64(amount), testPeer.Peer) + } + expectedBalance := int64(cnt * amount) + realBalance := swap.balances[testPeer.ID()] + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After %d credits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance)) + } + + testPeer2 := newDummyPeer() + amount = mrand.Intn(100) + cnt = 1 + mrand.Intn(10) + for i := 0; i < cnt; i++ { + swap.Add(0-int64(amount), testPeer2.Peer) + } + expectedBalance = int64(0 - (cnt * amount)) + realBalance = swap.balances[testPeer2.ID()] + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After %d debits of %d, expected balance to be: %d, but is: %d", cnt, amount, expectedBalance, realBalance)) + } + + //mixed debits and credits + amount1 := mrand.Intn(100) + amount2 := mrand.Intn(55) + amount3 := mrand.Intn(999) + swap.Add(int64(amount1), testPeer2.Peer) + swap.Add(int64(0-amount2), testPeer2.Peer) + swap.Add(int64(0-amount3), testPeer2.Peer) + + expectedBalance = expectedBalance + int64(amount1-amount2-amount3) + realBalance = swap.balances[testPeer2.ID()] + + if expectedBalance != realBalance { + t.Fatal(fmt.Sprintf("After mixed debits and credits, expected balance to be: %d, but is: %d", expectedBalance, realBalance)) + } +} + +//try restoring a balance from state store +//this is simulated by creating a node, +//assigning it an arbitrary balance, +//then closing the state store. +//Then we re-open the state store and check that +//the balance is still the same +func TestRestoreBalanceFromStateStore(t *testing.T) { + //create a test swap account + swap, testDir := createTestSwap(t) + defer os.RemoveAll(testDir) + + testPeer := newDummyPeer() + swap.balances[testPeer.ID()] = -8888 + + tmpBalance := swap.balances[testPeer.ID()] + swap.stateStore.Put(testPeer.ID().String(), &tmpBalance) + + swap.stateStore.Close() + swap.stateStore = nil + + stateStore, err := state.NewDBStore(testDir) + if err != nil { + t.Fatal(err) + } + + var newBalance int64 + stateStore.Get(testPeer.ID().String(), &newBalance) + + //compare the balances + if tmpBalance != newBalance { + t.Fatal(fmt.Sprintf("Unexpected balance value after sending cheap message test. Expected balance: %d, balance is: %d", + tmpBalance, newBalance)) + } +} + +//create a test swap account +//creates a stateStore for persistence and a Swap account +func createTestSwap(t *testing.T) (*Swap, string) { + dir, err := ioutil.TempDir("", "swap_test_store") + if err != nil { + t.Fatal(err) + } + stateStore, err2 := state.NewDBStore(dir) + if err2 != nil { + t.Fatal(err2) + } + swap := New(stateStore) + return swap, dir +} + +type dummyPeer struct { + *protocols.Peer +} + +//creates a dummy protocols.Peer with dummy MsgReadWriter +func newDummyPeer() *dummyPeer { + id := adapters.RandomNodeConfig().ID + protoPeer := protocols.NewPeer(p2p.NewPeer(id, "testPeer", nil), nil, nil) + dummy := &dummyPeer{ + Peer: protoPeer, + } + return dummy +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 8b26615299..dc3756d3af 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -49,8 +49,9 @@ import ( "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/storage/mock" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/swap" "github.com/ethereum/go-ethereum/swarm/tracing" ) @@ -78,6 +79,7 @@ type Swarm struct { netStore *storage.NetStore sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit ps *pss.Pss + swap *swap.Swap tracerClose io.Closer } @@ -171,34 +173,60 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e delivery := stream.NewDelivery(to, self.netStore) self.netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, config.DeliverySkipCheck).New + if config.SwapEnabled { + balancesStore, err := state.NewDBStore(filepath.Join(config.Path, "balances.db")) + if err != nil { + return nil, err + } + self.swap = swap.New(balancesStore) + } + var nodeID enode.ID if err := nodeID.UnmarshalText([]byte(config.NodeID)); err != nil { return nil, err } - self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, &stream.RegistryOptions{ + + syncing := stream.SyncingAutoSubscribe + if !config.SyncEnabled || config.LightNodeEnabled { + syncing = stream.SyncingDisabled + } + + retrieval := stream.RetrievalEnabled + if config.LightNodeEnabled { + retrieval = stream.RetrievalClientOnly + } + + registryOptions := &stream.RegistryOptions{ SkipCheck: config.DeliverySkipCheck, - DoSync: config.SyncEnabled, - DoRetrieve: true, + Syncing: syncing, + Retrieval: retrieval, SyncUpdateDelay: config.SyncUpdateDelay, - }) + MaxPeerServers: config.MaxStreamPeerServers, + } + self.streamer = stream.NewRegistry(nodeID, delivery, self.netStore, stateStore, registryOptions, self.swap) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) - var resourceHandler *mru.Handler - rhparams := &mru.HandlerParams{} + var feedsHandler *feed.Handler + fhParams := &feed.HandlerParams{} - resourceHandler = mru.NewHandler(rhparams) - resourceHandler.SetStore(self.netStore) + feedsHandler = feed.NewHandler(fhParams) + feedsHandler.SetStore(self.netStore) lstore.Validators = []storage.ChunkValidator{ storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)), - resourceHandler, + feedsHandler, + } + + err = lstore.Migrate() + if err != nil { + return nil, err } log.Debug("Setup local storage") - self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) + self.bzz = network.NewBzz(bzzconfig, to, stateStore, self.streamer.GetSpec(), self.streamer.Run) // Pss = postal service over swarm (devp2p over bzz) self.ps, err = pss.NewPss(to, config.Pss) @@ -209,7 +237,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) } - self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey) + self.api = api.NewAPI(self.fileStore, self.dns, feedsHandler, self.privateKey) self.sfs = fuse.NewSwarmFS(self.api) log.Debug("Initialized FUSE filesystem") @@ -335,7 +363,9 @@ func (self *Swarm) Start(srv *p2p.Server) error { newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String())) log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) // set chequebook - if self.config.SwapEnabled { + //TODO: Currently if swap is enabled and no chequebook (or inexistent) contract is provided, the node would crash. + //Once we integrate back the contracts, this check MUST be revisited + if self.config.SwapEnabled && self.config.SwapAPI != "" { ctx := context.Background() // The initial setup has no deadline. err := self.SetChequebook(ctx) if err != nil { diff --git a/swarm/swarm_test.go b/swarm/swarm_test.go index c6569e37bf..d85eb91185 100644 --- a/swarm/swarm_test.go +++ b/swarm/swarm_test.go @@ -316,11 +316,11 @@ func TestLocalStoreAndRetrieve(t *testing.T) { } // by default, test only the lonely chunk cases - sizes := []int{1, 60, 4097, 524288 + 1, 7*524288 + 1, 128*524288 + 1} + sizes := []int{1, 60, 4097, 524288 + 1, 7*524288 + 1} if *longrunning { // test broader set of cases if -longruning flag is set - sizes = append(sizes, 83, 179, 253, 1024, 4095, 4096, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678, 67298391, 524288, 524288+4096, 524288+4097, 7*524288, 7*524288+4096, 7*524288+4097, 128*524288, 128*524288+4096, 128*524288+4097, 816778334) + sizes = append(sizes, 83, 179, 253, 1024, 4095, 4096, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678, 67298391, 524288, 524288+4096, 524288+4097, 7*524288, 7*524288+4096, 7*524288+4097, 128*524288+1, 128*524288, 128*524288+4096, 128*524288+4097, 816778334) } for _, n := range sizes { testLocalStoreAndRetrieve(t, swarm, n, true) diff --git a/swarm/testutil/file.go b/swarm/testutil/file.go index ecb0d971ed..70732aa92e 100644 --- a/swarm/testutil/file.go +++ b/swarm/testutil/file.go @@ -17,8 +17,10 @@ package testutil import ( + "bytes" "io" "io/ioutil" + "math/rand" "os" "strings" "testing" @@ -42,3 +44,22 @@ func TempFileWithContent(t *testing.T, content string) string { } return tempFile.Name() } + +// RandomBytes returns pseudo-random deterministic result +// because test fails must be reproducible +func RandomBytes(seed, length int) []byte { + b := make([]byte, length) + reader := rand.New(rand.NewSource(int64(seed))) + for n := 0; n < length; { + read, err := reader.Read(b[n:]) + if err != nil { + panic(err) + } + n += read + } + return b +} + +func RandomReader(seed, length int) *bytes.Reader { + return bytes.NewReader(RandomBytes(seed, length)) +} diff --git a/swarm/tracing/tracing.go b/swarm/tracing/tracing.go index b84cfb3102..f95fa41b8f 100644 --- a/swarm/tracing/tracing.go +++ b/swarm/tracing/tracing.go @@ -9,7 +9,6 @@ import ( "github.com/ethereum/go-ethereum/log" jaeger "github.com/uber/jaeger-client-go" jaegercfg "github.com/uber/jaeger-client-go/config" - jaegerlog "github.com/uber/jaeger-client-go/log" cli "gopkg.in/urfave/cli.v1" ) @@ -85,13 +84,13 @@ func initTracer(endpoint, svc string) (closer io.Closer) { // Example logger and metrics factory. Use github.com/uber/jaeger-client-go/log // and github.com/uber/jaeger-lib/metrics respectively to bind to real logging and metrics // frameworks. - jLogger := jaegerlog.StdLogger + //jLogger := jaegerlog.StdLogger //jMetricsFactory := metrics.NullFactory // Initialize tracer with a logger and a metrics factory closer, err := cfg.InitGlobalTracer( svc, - jaegercfg.Logger(jLogger), + //jaegercfg.Logger(jLogger), //jaegercfg.Metrics(jMetricsFactory), //jaegercfg.Observer(rpcmetrics.NewObserver(jMetricsFactory, rpcmetrics.DefaultNameNormalizer)), ) diff --git a/swarm/version/version.go b/swarm/version/version.go index e2804b9756..17ef34f5f4 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 0 // Major version component of the current release VersionMinor = 3 // Minor version component of the current release - VersionPatch = 5 // Patch version component of the current release + VersionPatch = 7 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/tests/block_test.go b/tests/block_test.go index c911199299..711a3f8695 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -30,11 +30,19 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcForgedTest/bcForkUncle\.json`) bt.skipLoad(`^bcMultiChainTest/(ChainAtoChainB_blockorder|CallContractFromNotBestBlock)`) bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) - // This test is broken - bt.fails(`blockhashNonConstArg_Constantinople`, "Broken test") + // Slow tests + bt.slow(`^bcExploitTest/DelegateCallSpam.json`) + bt.slow(`^bcExploitTest/ShanghaiLove.json`) + bt.slow(`^bcExploitTest/SuicideIssue.json`) + bt.slow(`^bcForkStressTest/`) + bt.slow(`^bcGasPricerTest/RPC_API_Test.json`) + bt.slow(`^bcWalletTest/`) - // Still failing tests - // bt.skipLoad(`^bcWalletTest.*_Byzantium$`) + // Still failing tests that we need to look into + //bt.fails(`^bcStateTests/suicideThenCheckBalance.json/suicideThenCheckBalance_Constantinople`, "TODO: investigate") + //bt.fails(`^bcStateTests/suicideStorageCheckVCreate2.json/suicideStorageCheckVCreate2_Constantinople`, "TODO: investigate") + //bt.fails(`^bcStateTests/suicideStorageCheckVCreate.json/suicideStorageCheckVCreate_Constantinople`, "TODO: investigate") + //bt.fails(`^bcStateTests/suicideStorageCheck.json/suicideStorageCheck_Constantinople`, "TODO: investigate") bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { if err := bt.checkFailure(t, name, test.Run()); err != nil { diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 427a94958a..9fa69bf4ed 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" @@ -48,12 +49,13 @@ func (t *BlockTest) UnmarshalJSON(in []byte) error { } type btJSON struct { - Blocks []btBlock `json:"blocks"` - Genesis btHeader `json:"genesisBlockHeader"` - Pre core.GenesisAlloc `json:"pre"` - Post core.GenesisAlloc `json:"postState"` - BestBlock common.UnprefixedHash `json:"lastblockhash"` - Network string `json:"network"` + Blocks []btBlock `json:"blocks"` + Genesis btHeader `json:"genesisBlockHeader"` + Pre core.GenesisAlloc `json:"pre"` + Post core.GenesisAlloc `json:"postState"` + BestBlock common.UnprefixedHash `json:"lastblockhash"` + Network string `json:"network"` + SealEngine string `json:"sealEngine"` } type btBlock struct { @@ -110,8 +112,13 @@ func (t *BlockTest) Run() error { if gblock.Root() != t.json.Genesis.StateRoot { return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", gblock.Root().Bytes()[:6], t.json.Genesis.StateRoot[:6]) } - - chain, err := core.NewBlockChain(db, nil, config, ethash.NewShared(), vm.Config{}, nil) + var engine consensus.Engine + if t.json.SealEngine == "NoProof" { + engine = ethash.NewFaker() + } else { + engine = ethash.NewShared() + } + chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieCleanLimit: 0}, config, engine, vm.Config{}, nil) if err != nil { return err } diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 20294cc9d5..fde9db3ad4 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -17,9 +17,8 @@ package tests import ( - "testing" - "math/big" + "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" @@ -71,6 +70,9 @@ func TestDifficulty(t *testing.T) { dt.config("Frontier", *params.TestnetChainConfig) dt.config("MainNetwork", mainnetChainConfig) dt.config("CustomMainNetwork", mainnetChainConfig) + dt.config("Constantinople", params.ChainConfig{ + ConstantinopleBlock: big.NewInt(0), + }) dt.config("difficulty.json", mainnetChainConfig) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, test *DifficultyTest) { diff --git a/tests/init_test.go b/tests/init_test.go index 90a74448a8..053cbd6fcd 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -25,6 +25,7 @@ import ( "path/filepath" "reflect" "regexp" + "runtime" "sort" "strings" "testing" @@ -90,7 +91,7 @@ type testMatcher struct { configpat []testConfig failpat []testFailure skiploadpat []*regexp.Regexp - skipshortpat []*regexp.Regexp + slowpat []*regexp.Regexp whitelistpat *regexp.Regexp } @@ -105,8 +106,8 @@ type testFailure struct { } // skipShortMode skips tests matching when the -short flag is used. -func (tm *testMatcher) skipShortMode(pattern string) { - tm.skipshortpat = append(tm.skipshortpat, regexp.MustCompile(pattern)) +func (tm *testMatcher) slow(pattern string) { + tm.slowpat = append(tm.slowpat, regexp.MustCompile(pattern)) } // skipLoad skips JSON loading of tests matching the pattern. @@ -133,11 +134,15 @@ func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) { // findSkip matches name against test skip patterns. func (tm *testMatcher) findSkip(name string) (reason string, skipload bool) { - if testing.Short() { - for _, re := range tm.skipshortpat { - if re.MatchString(name) { + isWin32 := runtime.GOARCH == "386" && runtime.GOOS == "windows" + for _, re := range tm.slowpat { + if re.MatchString(name) { + if testing.Short() { return "skipped in -short mode", false } + if isWin32 { + return "skipped on 32bit windows", false + } } } for _, re := range tm.skiploadpat { diff --git a/tests/state_test.go b/tests/state_test.go index 91c9a9f447..ad77e4f339 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -30,21 +30,26 @@ func TestState(t *testing.T) { st := new(testMatcher) // Long tests: - st.skipShortMode(`^stQuadraticComplexityTest/`) + st.slow(`^stAttackTest/ContractCreationSpam`) + st.slow(`^stBadOpcode/badOpcodes`) + st.slow(`^stPreCompiledContracts/modexp`) + st.slow(`^stQuadraticComplexityTest/`) + st.slow(`^stStaticCall/static_Call50000`) + st.slow(`^stStaticCall/static_Return50000`) + st.slow(`^stStaticCall/static_Call1MB`) + st.slow(`^stSystemOperationsTest/CallRecursiveBomb`) + st.slow(`^stTransactionTest/Opcodes_TransactionInit`) // Broken tests: st.skipLoad(`^stTransactionTest/OverflowGasRequire\.json`) // gasLimit > 256 bits st.skipLoad(`^stTransactionTest/zeroSigTransa[^/]*\.json`) // EIP-86 is not supported yet // Expected failures: st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch.json/Constantinople`, "bug in test") st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { subtest := subtest - if subtest.Fork == "Constantinople" { - // Skipping constantinople due to net sstore gas changes affecting all tests - continue - } key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key t.Run(key, func(t *testing.T) { diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 5d2251e529..3683aae320 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -143,9 +143,6 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD if _, _, _, err := core.ApplyMessage(evm, msg, gaspool); err != nil { statedb.RevertToSnapshot(snapshot) } - if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { - return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) - } // Commit block statedb.Commit(config.IsEIP158(block.Number())) // Add 0-value mining reward. This only makes a difference in the cases @@ -161,6 +158,9 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD if root != common.Hash(post.Root) { return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } + if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { + return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) + } return statedb, nil } diff --git a/tests/testdata b/tests/testdata index ad2184adca..95a3092038 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit ad2184adca367c0b68c65b44519dba16e1d0b9e2 +Subproject commit 95a309203890e6244c6d4353ca411671973c13b5 diff --git a/tests/vm_test.go b/tests/vm_test.go index c9f5e225e9..441483dffa 100644 --- a/tests/vm_test.go +++ b/tests/vm_test.go @@ -25,13 +25,9 @@ import ( func TestVM(t *testing.T) { t.Parallel() vmt := new(testMatcher) + vmt.slow("^vmPerformance") vmt.fails("^vmSystemOperationsTest.json/createNameRegistrator$", "fails without parallel execution") - vmt.skipLoad(`^vmInputLimits(Light)?.json`) // log format broken - - vmt.skipShortMode("^vmPerformanceTest.json") - vmt.skipShortMode("^vmInputLimits(Light)?.json") - vmt.walk(t, vmTestDir, func(t *testing.T, name string, test *VMTest) { withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { return vmt.checkFailure(t, name, test.Run(vmconfig)) diff --git a/trie/database.go b/trie/database.go index d0691b637e..71190b3f3f 100644 --- a/trie/database.go +++ b/trie/database.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/allegro/bigcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -30,6 +31,11 @@ import ( ) var ( + memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) + memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) + memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) + memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) @@ -64,9 +70,10 @@ type DatabaseReader interface { type Database struct { diskdb ethdb.Database // Persistent storage for matured trie nodes - nodes map[common.Hash]*cachedNode // Data and references relationships of a node - oldest common.Hash // Oldest tracked node, flush-list head - newest common.Hash // Newest tracked node, flush-list tail + cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs + dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes + oldest common.Hash // Oldest tracked node, flush-list head + newest common.Hash // Newest tracked node, flush-list tail preimages map[common.Hash][]byte // Preimages of nodes from the secure trie seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys @@ -79,7 +86,7 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - nodesSize common.StorageSize // Storage size of the nodes cache (exc. flushlist) + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. flushlist) preimagesSize common.StorageSize // Storage size of the preimages cache lock sync.RWMutex @@ -262,11 +269,30 @@ func expandNode(hash hashNode, n node, cachegen uint16) node { } // NewDatabase creates a new trie database to store ephemeral trie content before -// its written out to disk or garbage collected. +// its written out to disk or garbage collected. No read cache is created, so all +// data retrievals will hit the underlying disk database. func NewDatabase(diskdb ethdb.Database) *Database { + return NewDatabaseWithCache(diskdb, 0) +} + +// NewDatabaseWithCache creates a new trie database to store ephemeral trie content +// before its written out to disk or garbage collected. It also acts as a read cache +// for nodes loaded from disk. +func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { + var cleans *bigcache.BigCache + if cache > 0 { + cleans, _ = bigcache.NewBigCache(bigcache.Config{ + Shards: 1024, + LifeWindow: time.Hour, + MaxEntriesInWindow: cache * 1024, + MaxEntrySize: 512, + HardMaxCacheSize: cache, + }) + } return &Database{ diskdb: diskdb, - nodes: map[common.Hash]*cachedNode{{}: {}}, + cleans: cleans, + dirties: map[common.Hash]*cachedNode{{}: {}}, preimages: make(map[common.Hash][]byte), } } @@ -293,7 +319,7 @@ func (db *Database) InsertBlob(hash common.Hash, blob []byte) { // size tracking. func (db *Database) insert(hash common.Hash, blob []byte, node node) { // If the node's already cached, skip - if _, ok := db.nodes[hash]; ok { + if _, ok := db.dirties[hash]; ok { return } // Create the cached entry for this node @@ -303,19 +329,19 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) { flushPrev: db.newest, } for _, child := range entry.childs() { - if c := db.nodes[child]; c != nil { + if c := db.dirties[child]; c != nil { c.parents++ } } - db.nodes[hash] = entry + db.dirties[hash] = entry // Update the flush-list endpoints if db.oldest == (common.Hash{}) { db.oldest, db.newest = hash, hash } else { - db.nodes[db.newest].flushNext, db.newest = hash, hash + db.dirties[db.newest].flushNext, db.newest = hash, hash } - db.nodesSize += common.StorageSize(common.HashLength + entry.size) + db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } // insertPreimage writes a new trie node pre-image to the memory database if it's @@ -333,35 +359,64 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash, cachegen uint16) node { - // Retrieve the node from cache if available + // Retrieve the node from the clean cache if available + if db.cleans != nil { + if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return mustDecodeNode(hash[:], enc, cachegen) + } + } + // Retrieve the node from the dirty cache if available db.lock.RLock() - node := db.nodes[hash] + dirty := db.dirties[hash] db.lock.RUnlock() - if node != nil { - return node.obj(hash, cachegen) + if dirty != nil { + return dirty.obj(hash, cachegen) } // Content unavailable in memory, attempt to retrieve from disk enc, err := db.diskdb.Get(hash[:]) if err != nil || enc == nil { return nil } + if db.cleans != nil { + db.cleans.Set(string(hash[:]), enc) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(enc))) + } return mustDecodeNode(hash[:], enc, cachegen) } // Node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. func (db *Database) Node(hash common.Hash) ([]byte, error) { - // Retrieve the node from cache if available + // Retrieve the node from the clean cache if available + if db.cleans != nil { + if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return enc, nil + } + } + // Retrieve the node from the dirty cache if available db.lock.RLock() - node := db.nodes[hash] + dirty := db.dirties[hash] db.lock.RUnlock() - if node != nil { - return node.rlp(), nil + if dirty != nil { + return dirty.rlp(), nil } // Content unavailable in memory, attempt to retrieve from disk - return db.diskdb.Get(hash[:]) + enc, err := db.diskdb.Get(hash[:]) + if err == nil && enc != nil { + if db.cleans != nil { + db.cleans.Set(string(hash[:]), enc) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(enc))) + } + } + return enc, err } // preimage retrieves a cached trie node pre-image from memory. If it cannot be @@ -395,8 +450,8 @@ func (db *Database) Nodes() []common.Hash { db.lock.RLock() defer db.lock.RUnlock() - var hashes = make([]common.Hash, 0, len(db.nodes)) - for hash := range db.nodes { + var hashes = make([]common.Hash, 0, len(db.dirties)) + for hash := range db.dirties { if hash != (common.Hash{}) { // Special case for "root" references/nodes hashes = append(hashes, hash) } @@ -415,18 +470,18 @@ func (db *Database) Reference(child common.Hash, parent common.Hash) { // reference is the private locked version of Reference. func (db *Database) reference(child common.Hash, parent common.Hash) { // If the node does not exist, it's a node pulled from disk, skip - node, ok := db.nodes[child] + node, ok := db.dirties[child] if !ok { return } // If the reference already exists, only duplicate for roots - if db.nodes[parent].children == nil { - db.nodes[parent].children = make(map[common.Hash]uint16) - } else if _, ok = db.nodes[parent].children[child]; ok && parent != (common.Hash{}) { + if db.dirties[parent].children == nil { + db.dirties[parent].children = make(map[common.Hash]uint16) + } else if _, ok = db.dirties[parent].children[child]; ok && parent != (common.Hash{}) { return } node.parents++ - db.nodes[parent].children[child]++ + db.dirties[parent].children[child]++ } // Dereference removes an existing reference from a root node. @@ -439,25 +494,25 @@ func (db *Database) Dereference(root common.Hash) { db.lock.Lock() defer db.lock.Unlock() - nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() db.dereference(root, common.Hash{}) - db.gcnodes += uint64(nodes - len(db.nodes)) - db.gcsize += storage - db.nodesSize + db.gcnodes += uint64(nodes - len(db.dirties)) + db.gcsize += storage - db.dirtiesSize db.gctime += time.Since(start) memcacheGCTimeTimer.Update(time.Since(start)) - memcacheGCSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheGCNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheGCSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheGCNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) } // dereference is the private locked version of Dereference. func (db *Database) dereference(child common.Hash, parent common.Hash) { // Dereference the parent-child - node := db.nodes[parent] + node := db.dirties[parent] if node.children != nil && node.children[child] > 0 { node.children[child]-- @@ -466,7 +521,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { } } // If the child does not exist, it's a previously committed node. - node, ok := db.nodes[child] + node, ok := db.dirties[child] if !ok { return } @@ -483,20 +538,20 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { switch child { case db.oldest: db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = common.Hash{} case db.newest: db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = common.Hash{} default: - db.nodes[node.flushPrev].flushNext = node.flushNext - db.nodes[node.flushNext].flushPrev = node.flushPrev + db.dirties[node.flushPrev].flushNext = node.flushNext + db.dirties[node.flushNext].flushPrev = node.flushPrev } // Dereference all children and delete the node for _, hash := range node.childs() { db.dereference(hash, child) } - delete(db.nodes, child) - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, child) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } } @@ -509,13 +564,13 @@ func (db *Database) Cap(limit common.StorageSize) error { // by only uncaching existing data when the database write finalizes. db.lock.RLock() - nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() batch := db.diskdb.NewBatch() - // db.nodesSize only contains the useful data in the cache, but when reporting + // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength) + size := db.dirtiesSize + common.StorageSize((len(db.dirties)-1)*2*common.HashLength) // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. @@ -540,7 +595,7 @@ func (db *Database) Cap(limit common.StorageSize) error { oldest := db.oldest for size > limit && oldest != (common.Hash{}) { // Fetch the oldest referenced node and push into the batch - node := db.nodes[oldest] + node := db.dirties[oldest] if err := batch.Put(oldest[:], node.rlp()); err != nil { db.lock.RUnlock() return err @@ -578,25 +633,25 @@ func (db *Database) Cap(limit common.StorageSize) error { db.preimagesSize = 0 } for db.oldest != oldest { - node := db.nodes[db.oldest] - delete(db.nodes, db.oldest) + node := db.dirties[db.oldest] + delete(db.dirties, db.oldest) db.oldest = node.flushNext - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } if db.oldest != (common.Hash{}) { - db.nodes[db.oldest].flushPrev = common.Hash{} + db.dirties[db.oldest].flushPrev = common.Hash{} } - db.flushnodes += uint64(nodes - len(db.nodes)) - db.flushsize += storage - db.nodesSize + db.flushnodes += uint64(nodes - len(db.dirties)) + db.flushsize += storage - db.dirtiesSize db.flushtime += time.Since(start) memcacheFlushTimeTimer.Update(time.Since(start)) - memcacheFlushSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheFlushNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheFlushSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheFlushNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), - "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), + "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) return nil } @@ -630,7 +685,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { } } // Move the trie itself into the batch, flushing if enough data is accumulated - nodes, storage := len(db.nodes), db.nodesSize + nodes, storage := len(db.dirties), db.dirtiesSize if err := db.commit(node, batch); err != nil { log.Error("Failed to commit trie from trie database", "err", err) db.lock.RUnlock() @@ -654,15 +709,15 @@ func (db *Database) Commit(node common.Hash, report bool) error { db.uncache(node) memcacheCommitTimeTimer.Update(time.Since(start)) - memcacheCommitSizeMeter.Mark(int64(storage - db.nodesSize)) - memcacheCommitNodesMeter.Mark(int64(nodes - len(db.nodes))) + memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) + memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) logger := log.Info if !report { logger = log.Debug } - logger("Persisted trie from memory database", "nodes", nodes-len(db.nodes)+int(db.flushnodes), "size", storage-db.nodesSize+db.flushsize, "time", time.Since(start)+db.flushtime, - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) + logger("Persisted trie from memory database", "nodes", nodes-len(db.dirties)+int(db.flushnodes), "size", storage-db.dirtiesSize+db.flushsize, "time", time.Since(start)+db.flushtime, + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) // Reset the garbage collection statistics db.gcnodes, db.gcsize, db.gctime = 0, 0, 0 @@ -674,7 +729,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { // commit is the private locked version of Commit. func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // If the node does not exist, it's a previously committed node - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return nil } @@ -702,7 +757,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // to disk. func (db *Database) uncache(hash common.Hash) { // If the node does not exist, we're done on this path - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return } @@ -710,20 +765,20 @@ func (db *Database) uncache(hash common.Hash) { switch hash { case db.oldest: db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = common.Hash{} case db.newest: db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = common.Hash{} default: - db.nodes[node.flushPrev].flushNext = node.flushNext - db.nodes[node.flushNext].flushPrev = node.flushPrev + db.dirties[node.flushPrev].flushNext = node.flushNext + db.dirties[node.flushNext].flushPrev = node.flushPrev } // Uncache the node's subtries and remove the node itself too for _, child := range node.childs() { db.uncache(child) } - delete(db.nodes, hash) - db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, hash) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } // Size returns the current storage size of the memory cache in front of the @@ -732,11 +787,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { db.lock.RLock() defer db.lock.RUnlock() - // db.nodesSize only contains the useful data in the cache, but when reporting + // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength) - return db.nodesSize + flushlistSize, db.preimagesSize + var flushlistSize = common.StorageSize((len(db.dirties) - 1) * 2 * common.HashLength) + return db.dirtiesSize + flushlistSize, db.preimagesSize } // verifyIntegrity is a debug method to iterate over the entire trie stored in @@ -749,12 +804,12 @@ func (db *Database) verifyIntegrity() { // Iterate over all the cached nodes and accumulate them into a set reachable := map[common.Hash]struct{}{{}: {}} - for child := range db.nodes[common.Hash{}].children { + for child := range db.dirties[common.Hash{}].children { db.accumulate(child, reachable) } // Find any unreachable but cached nodes unreachable := []string{} - for hash, node := range db.nodes { + for hash, node := range db.dirties { if _, ok := reachable[hash]; !ok { unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}", hash, node.node, node.parents, node.flushPrev, node.flushNext)) @@ -769,7 +824,7 @@ func (db *Database) verifyIntegrity() { // cached children found in memory. func (db *Database) accumulate(hash common.Hash, reachable map[common.Hash]struct{}) { // Mark the node reachable if present in the memory cache - node, ok := db.nodes[hash] + node, ok := db.dirties[hash] if !ok { return } diff --git a/trie/iterator.go b/trie/iterator.go index 00b890eb89..77f1681665 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -181,6 +181,8 @@ func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { hasher := newHasher(0, 0, nil) + defer returnHasherToPool(hasher) + proofs := make([][]byte, 0, len(it.stack)) for i, item := range it.stack[:len(it.stack)-1] { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 2a510b1c2d..4f633b195f 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -113,7 +113,7 @@ func TestNodeIteratorCoverage(t *testing.T) { t.Errorf("failed to retrieve reported node %x: %v", hash, err) } } - for hash, obj := range db.nodes { + for hash, obj := range db.dirties { if obj != nil && hash != (common.Hash{}) { if _, ok := hashes[hash]; !ok { t.Errorf("state entry not reported %x", hash) @@ -333,8 +333,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } } if memonly { - robj = triedb.nodes[rkey] - delete(triedb.nodes, rkey) + robj = triedb.dirties[rkey] + delete(triedb.dirties, rkey) } else { rval, _ = diskdb.Get(rkey[:]) diskdb.Delete(rkey[:]) @@ -350,7 +350,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { // Add the node back and continue iteration. if memonly { - triedb.nodes[rkey] = robj + triedb.dirties[rkey] = robj } else { diskdb.Put(rkey[:], rval) } @@ -393,8 +393,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { barNodeObj *cachedNode ) if memonly { - barNodeObj = triedb.nodes[barNodeHash] - delete(triedb.nodes, barNodeHash) + barNodeObj = triedb.dirties[barNodeHash] + delete(triedb.dirties, barNodeHash) } else { barNodeBlob, _ = diskdb.Get(barNodeHash[:]) diskdb.Delete(barNodeHash[:]) @@ -411,7 +411,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Reinsert the missing node. if memonly { - triedb.nodes[barNodeHash] = barNodeObj + triedb.dirties[barNodeHash] = barNodeObj } else { diskdb.Put(barNodeHash[:], barNodeBlob) } diff --git a/trie/proof.go b/trie/proof.go index 6cb8f4d5f7..f90ecd7d88 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -66,6 +66,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { } } hasher := newHasher(0, 0, nil) + defer returnHasherToPool(hasher) + for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. diff --git a/trie/trie.go b/trie/trie.go index e920ccd23f..af424d4ac6 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -65,9 +65,8 @@ type LeafCallback func(leaf []byte, parent common.Hash) error // // Trie is not safe for concurrent use. type Trie struct { - db *Database - root node - originalRoot common.Hash + db *Database + root node // Cache generation values. // cachegen increases by one with each commit operation. @@ -98,8 +97,7 @@ func New(root common.Hash, db *Database) (*Trie, error) { panic("trie.New called without a database") } trie := &Trie{ - db: db, - originalRoot: root, + db: db, } if root != (common.Hash{}) && root != emptyRoot { rootnode, err := trie.resolveHash(root[:], nil) diff --git a/trie/trie_test.go b/trie/trie_test.go index f8e5fd12a1..f9d6029c99 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -119,7 +119,7 @@ func testMissingNode(t *testing.T, memonly bool) { hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9") if memonly { - delete(triedb.nodes, hash) + delete(triedb.dirties, hash) } else { diskdb.Delete(hash[:]) } @@ -342,15 +342,16 @@ func TestCacheUnload(t *testing.T) { // Commit the trie repeatedly and access key1. // The branch containing it is loaded from DB exactly two times: // in the 0th and 6th iteration. - db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} - trie, _ = New(root, NewDatabase(db)) + diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} + triedb := NewDatabase(diskdb) + trie, _ = New(root, triedb) trie.SetCacheLimit(5) for i := 0; i < 12; i++ { getString(trie, key1) trie.Commit(nil) } // Check that it got loaded two times. - for dbkey, count := range db.gets { + for dbkey, count := range diskdb.gets { if count != 2 { t.Errorf("db key %x loaded %d times, want %d times", []byte(dbkey), count, 2) } diff --git a/vendor/github.com/allegro/bigcache/LICENSE b/vendor/github.com/allegro/bigcache/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/vendor/github.com/allegro/bigcache/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/allegro/bigcache/README.md b/vendor/github.com/allegro/bigcache/README.md new file mode 100644 index 0000000000..c23f7f36ce --- /dev/null +++ b/vendor/github.com/allegro/bigcache/README.md @@ -0,0 +1,150 @@ +# BigCache [![Build Status](https://travis-ci.org/allegro/bigcache.svg?branch=master)](https://travis-ci.org/allegro/bigcache) [![Coverage Status](https://coveralls.io/repos/github/allegro/bigcache/badge.svg?branch=master)](https://coveralls.io/github/allegro/bigcache?branch=master) [![GoDoc](https://godoc.org/github.com/allegro/bigcache?status.svg)](https://godoc.org/github.com/allegro/bigcache) [![Go Report Card](https://goreportcard.com/badge/github.com/allegro/bigcache)](https://goreportcard.com/report/github.com/allegro/bigcache) + +Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance. +BigCache keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place, +therefore entries (de)serialization in front of the cache will be needed in most use cases. + +## Usage + +### Simple initialization + +```go +import "github.com/allegro/bigcache" + +cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute)) + +cache.Set("my-unique-key", []byte("value")) + +entry, _ := cache.Get("my-unique-key") +fmt.Println(string(entry)) +``` + +### Custom initialization + +When cache load can be predicted in advance then it is better to use custom initialization because additional memory +allocation can be avoided in that way. + +```go +import ( + "log" + + "github.com/allegro/bigcache" +) + +config := bigcache.Config { + // number of shards (must be a power of 2) + Shards: 1024, + // time after which entry can be evicted + LifeWindow: 10 * time.Minute, + // rps * lifeWindow, used only in initial memory allocation + MaxEntriesInWindow: 1000 * 10 * 60, + // max entry size in bytes, used only in initial memory allocation + MaxEntrySize: 500, + // prints information about additional memory allocation + Verbose: true, + // cache will not allocate more memory than this limit, value in MB + // if value is reached then the oldest entries can be overridden for the new ones + // 0 value means no size limit + HardMaxCacheSize: 8192, + // callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A bitmask representing the reason will be returned. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + OnRemove: nil, + // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // Ignored if OnRemove is specified. + OnRemoveWithReason: nil, + } + +cache, initErr := bigcache.NewBigCache(config) +if initErr != nil { + log.Fatal(initErr) +} + +cache.Set("my-unique-key", []byte("value")) + +if entry, err := cache.Get("my-unique-key"); err == nil { + fmt.Println(string(entry)) +} +``` + +## Benchmarks + +Three caches were compared: bigcache, [freecache](https://github.com/coocood/freecache) and map. +Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10. + +### Writes and reads + +```bash +cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m + +BenchmarkMapSet-8 3000000 569 ns/op 202 B/op 3 allocs/op +BenchmarkConcurrentMapSet-8 1000000 1592 ns/op 347 B/op 8 allocs/op +BenchmarkFreeCacheSet-8 3000000 775 ns/op 355 B/op 2 allocs/op +BenchmarkBigCacheSet-8 3000000 640 ns/op 303 B/op 2 allocs/op +BenchmarkMapGet-8 5000000 407 ns/op 24 B/op 1 allocs/op +BenchmarkConcurrentMapGet-8 3000000 558 ns/op 24 B/op 2 allocs/op +BenchmarkFreeCacheGet-8 2000000 682 ns/op 136 B/op 2 allocs/op +BenchmarkBigCacheGet-8 3000000 512 ns/op 152 B/op 4 allocs/op +BenchmarkBigCacheSetParallel-8 10000000 225 ns/op 313 B/op 3 allocs/op +BenchmarkFreeCacheSetParallel-8 10000000 218 ns/op 341 B/op 3 allocs/op +BenchmarkConcurrentMapSetParallel-8 5000000 318 ns/op 200 B/op 6 allocs/op +BenchmarkBigCacheGetParallel-8 20000000 178 ns/op 152 B/op 4 allocs/op +BenchmarkFreeCacheGetParallel-8 20000000 295 ns/op 136 B/op 3 allocs/op +BenchmarkConcurrentMapGetParallel-8 10000000 237 ns/op 24 B/op 2 allocs/op +``` + +Writes and reads in bigcache are faster than in freecache. +Writes to map are the slowest. + +### GC pause time + +```bash +cd caches_bench; go run caches_gc_overhead_comparison.go + +Number of entries: 20000000 +GC pause for bigcache: 5.8658ms +GC pause for freecache: 32.4341ms +GC pause for map: 52.9661ms +``` + +Test shows how long are the GC pauses for caches filled with 20mln of entries. +Bigcache and freecache have very similar GC pause time. +It is clear that both reduce GC overhead in contrast to map +which GC pause time took more than 10 seconds. + +## How it works + +BigCache relies on optimization presented in 1.5 version of Go ([issue-9477](https://github.com/golang/go/issues/9477)). +This optimization states that if map without pointers in keys and values is used then GC will omit its content. +Therefore BigCache uses `map[uint64]uint32` where keys are hashed and values are offsets of entries. + +Entries are kept in bytes array, to omit GC again. +Bytes array size can grow to gigabytes without impact on performance +because GC will only see single pointer to it. + +## Bigcache vs Freecache + +Both caches provide the same core features but they reduce GC overhead in different ways. +Bigcache relies on `map[uint64]uint32`, freecache implements its own mapping built on +slices to reduce number of pointers. + +Results from benchmark tests are presented above. +One of the advantage of bigcache over freecache is that you don’t need to know +the size of the cache in advance, because when bigcache is full, +it can allocate additional memory for new entries instead of +overwriting existing ones as freecache does currently. +However hard max size in bigcache also can be set, check [HardMaxCacheSize](https://godoc.org/github.com/allegro/bigcache#Config). + +## HTTP Server + +This package also includes an easily deployable HTTP implementation of BigCache, which can be found in the [server](/server) package. + +## More + +Bigcache genesis is described in allegro.tech blog post: [writing a very fast cache service in Go](http://allegro.tech/2016/03/writing-fast-cache-service-in-go.html) + +## License + +BigCache is released under the Apache 2.0 license (see [LICENSE](LICENSE)) diff --git a/vendor/github.com/allegro/bigcache/bigcache.go b/vendor/github.com/allegro/bigcache/bigcache.go new file mode 100644 index 0000000000..e6aee8663b --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bigcache.go @@ -0,0 +1,202 @@ +package bigcache + +import ( + "fmt" + "time" +) + +const ( + minimumEntriesInShard = 10 // Minimum number of entries in single shard +) + +// BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance. +// It keeps entries on heap but omits GC for them. To achieve that, operations take place on byte arrays, +// therefore entries (de)serialization in front of the cache will be needed in most use cases. +type BigCache struct { + shards []*cacheShard + lifeWindow uint64 + clock clock + hash Hasher + config Config + shardMask uint64 + maxShardSize uint32 + close chan struct{} +} + +// RemoveReason is a value used to signal to the user why a particular key was removed in the OnRemove callback. +type RemoveReason uint32 + +const ( + // Expired means the key is past its LifeWindow. + Expired RemoveReason = iota + // NoSpace means the key is the oldest and the cache size was at its maximum when Set was called, or the + // entry exceeded the maximum shard size. + NoSpace + // Deleted means Delete was called and this key was removed as a result. + Deleted +) + +// NewBigCache initialize new instance of BigCache +func NewBigCache(config Config) (*BigCache, error) { + return newBigCache(config, &systemClock{}) +} + +func newBigCache(config Config, clock clock) (*BigCache, error) { + + if !isPowerOfTwo(config.Shards) { + return nil, fmt.Errorf("Shards number must be power of two") + } + + if config.Hasher == nil { + config.Hasher = newDefaultHasher() + } + + cache := &BigCache{ + shards: make([]*cacheShard, config.Shards), + lifeWindow: uint64(config.LifeWindow.Seconds()), + clock: clock, + hash: config.Hasher, + config: config, + shardMask: uint64(config.Shards - 1), + maxShardSize: uint32(config.maximumShardSize()), + close: make(chan struct{}), + } + + var onRemove func(wrappedEntry []byte, reason RemoveReason) + if config.OnRemove != nil { + onRemove = cache.providedOnRemove + } else if config.OnRemoveWithReason != nil { + onRemove = cache.providedOnRemoveWithReason + } else { + onRemove = cache.notProvidedOnRemove + } + + for i := 0; i < config.Shards; i++ { + cache.shards[i] = initNewShard(config, onRemove, clock) + } + + if config.CleanWindow > 0 { + go func() { + ticker := time.NewTicker(config.CleanWindow) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + cache.cleanUp(uint64(t.Unix())) + case <-cache.close: + return + } + } + }() + } + + return cache, nil +} + +// Close is used to signal a shutdown of the cache when you are done with it. +// This allows the cleaning goroutines to exit and ensures references are not +// kept to the cache preventing GC of the entire cache. +func (c *BigCache) Close() error { + close(c.close) + return nil +} + +// Get reads entry for the key. +// It returns an EntryNotFoundError when +// no entry exists for the given key. +func (c *BigCache) Get(key string) ([]byte, error) { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.get(key, hashedKey) +} + +// Set saves entry under the key +func (c *BigCache) Set(key string, entry []byte) error { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.set(key, hashedKey, entry) +} + +// Delete removes the key +func (c *BigCache) Delete(key string) error { + hashedKey := c.hash.Sum64(key) + shard := c.getShard(hashedKey) + return shard.del(key, hashedKey) +} + +// Reset empties all cache shards +func (c *BigCache) Reset() error { + for _, shard := range c.shards { + shard.reset(c.config) + } + return nil +} + +// Len computes number of entries in cache +func (c *BigCache) Len() int { + var len int + for _, shard := range c.shards { + len += shard.len() + } + return len +} + +// Capacity returns amount of bytes store in the cache. +func (c *BigCache) Capacity() int { + var len int + for _, shard := range c.shards { + len += shard.capacity() + } + return len +} + +// Stats returns cache's statistics +func (c *BigCache) Stats() Stats { + var s Stats + for _, shard := range c.shards { + tmp := shard.getStats() + s.Hits += tmp.Hits + s.Misses += tmp.Misses + s.DelHits += tmp.DelHits + s.DelMisses += tmp.DelMisses + s.Collisions += tmp.Collisions + } + return s +} + +// Iterator returns iterator function to iterate over EntryInfo's from whole cache. +func (c *BigCache) Iterator() *EntryInfoIterator { + return newIterator(c) +} + +func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool { + oldestTimestamp := readTimestampFromEntry(oldestEntry) + if currentTimestamp-oldestTimestamp > c.lifeWindow { + evict(Expired) + return true + } + return false +} + +func (c *BigCache) cleanUp(currentTimestamp uint64) { + for _, shard := range c.shards { + shard.cleanUp(currentTimestamp) + } +} + +func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) { + return c.shards[hashedKey&c.shardMask] +} + +func (c *BigCache) providedOnRemove(wrappedEntry []byte, reason RemoveReason) { + c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry)) +} + +func (c *BigCache) providedOnRemoveWithReason(wrappedEntry []byte, reason RemoveReason) { + if c.config.onRemoveFilter == 0 || (1< 0 { + c.config.OnRemoveWithReason(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry), reason) + } +} + +func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte, reason RemoveReason) { +} diff --git a/vendor/github.com/allegro/bigcache/bytes.go b/vendor/github.com/allegro/bigcache/bytes.go new file mode 100644 index 0000000000..3944bfe135 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bytes.go @@ -0,0 +1,14 @@ +// +build !appengine + +package bigcache + +import ( + "reflect" + "unsafe" +) + +func bytesToString(b []byte) string { + bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len} + return *(*string)(unsafe.Pointer(&strHeader)) +} diff --git a/vendor/github.com/allegro/bigcache/bytes_appengine.go b/vendor/github.com/allegro/bigcache/bytes_appengine.go new file mode 100644 index 0000000000..3892f3b54f --- /dev/null +++ b/vendor/github.com/allegro/bigcache/bytes_appengine.go @@ -0,0 +1,7 @@ +// +build appengine + +package bigcache + +func bytesToString(b []byte) string { + return string(b) +} diff --git a/vendor/github.com/allegro/bigcache/clock.go b/vendor/github.com/allegro/bigcache/clock.go new file mode 100644 index 0000000000..f8b535e133 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/clock.go @@ -0,0 +1,14 @@ +package bigcache + +import "time" + +type clock interface { + epoch() int64 +} + +type systemClock struct { +} + +func (c systemClock) epoch() int64 { + return time.Now().Unix() +} diff --git a/vendor/github.com/allegro/bigcache/config.go b/vendor/github.com/allegro/bigcache/config.go new file mode 100644 index 0000000000..9654143ab1 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/config.go @@ -0,0 +1,86 @@ +package bigcache + +import "time" + +// Config for BigCache +type Config struct { + // Number of cache shards, value must be a power of two + Shards int + // Time after which entry can be evicted + LifeWindow time.Duration + // Interval between removing expired entries (clean up). + // If set to <= 0 then no action is performed. Setting to < 1 second is counterproductive — bigcache has a one second resolution. + CleanWindow time.Duration + // Max number of entries in life window. Used only to calculate initial size for cache shards. + // When proper value is set then additional memory allocation does not occur. + MaxEntriesInWindow int + // Max size of entry in bytes. Used only to calculate initial size for cache shards. + MaxEntrySize int + // Verbose mode prints information about new memory allocation + Verbose bool + // Hasher used to map between string keys and unsigned 64bit integers, by default fnv64 hashing is used. + Hasher Hasher + // HardMaxCacheSize is a limit for cache size in MB. Cache will not allocate more memory than this limit. + // It can protect application from consuming all available memory on machine, therefore from running OOM Killer. + // Default value is 0 which means unlimited size. When the limit is higher than 0 and reached then + // the oldest entries are overridden for the new ones. + HardMaxCacheSize int + // OnRemove is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + OnRemove func(key string, entry []byte) + // OnRemoveWithReason is a callback fired when the oldest entry is removed because of its expiration time or no space left + // for the new entry, or because delete was called. A constant representing the reason will be passed through. + // Default value is nil which means no callback and it prevents from unwrapping the oldest entry. + // Ignored if OnRemove is specified. + OnRemoveWithReason func(key string, entry []byte, reason RemoveReason) + + onRemoveFilter int + + // Logger is a logging interface and used in combination with `Verbose` + // Defaults to `DefaultLogger()` + Logger Logger +} + +// DefaultConfig initializes config with default values. +// When load for BigCache can be predicted in advance then it is better to use custom config. +func DefaultConfig(eviction time.Duration) Config { + return Config{ + Shards: 1024, + LifeWindow: eviction, + CleanWindow: 0, + MaxEntriesInWindow: 1000 * 10 * 60, + MaxEntrySize: 500, + Verbose: true, + Hasher: newDefaultHasher(), + HardMaxCacheSize: 0, + Logger: DefaultLogger(), + } +} + +// initialShardSize computes initial shard size +func (c Config) initialShardSize() int { + return max(c.MaxEntriesInWindow/c.Shards, minimumEntriesInShard) +} + +// maximumShardSize computes maximum shard size +func (c Config) maximumShardSize() int { + maxShardSize := 0 + + if c.HardMaxCacheSize > 0 { + maxShardSize = convertMBToBytes(c.HardMaxCacheSize) / c.Shards + } + + return maxShardSize +} + +// OnRemoveFilterSet sets which remove reasons will trigger a call to OnRemoveWithReason. +// Filtering out reasons prevents bigcache from unwrapping them, which saves cpu. +func (c Config) OnRemoveFilterSet(reasons ...RemoveReason) Config { + c.onRemoveFilter = 0 + for i := range reasons { + c.onRemoveFilter |= 1 << uint(reasons[i]) + } + + return c +} diff --git a/vendor/github.com/allegro/bigcache/encoding.go b/vendor/github.com/allegro/bigcache/encoding.go new file mode 100644 index 0000000000..4d434e5dce --- /dev/null +++ b/vendor/github.com/allegro/bigcache/encoding.go @@ -0,0 +1,62 @@ +package bigcache + +import ( + "encoding/binary" +) + +const ( + timestampSizeInBytes = 8 // Number of bytes used for timestamp + hashSizeInBytes = 8 // Number of bytes used for hash + keySizeInBytes = 2 // Number of bytes used for size of entry key + headersSizeInBytes = timestampSizeInBytes + hashSizeInBytes + keySizeInBytes // Number of bytes used for all headers +) + +func wrapEntry(timestamp uint64, hash uint64, key string, entry []byte, buffer *[]byte) []byte { + keyLength := len(key) + blobLength := len(entry) + headersSizeInBytes + keyLength + + if blobLength > len(*buffer) { + *buffer = make([]byte, blobLength) + } + blob := *buffer + + binary.LittleEndian.PutUint64(blob, timestamp) + binary.LittleEndian.PutUint64(blob[timestampSizeInBytes:], hash) + binary.LittleEndian.PutUint16(blob[timestampSizeInBytes+hashSizeInBytes:], uint16(keyLength)) + copy(blob[headersSizeInBytes:], key) + copy(blob[headersSizeInBytes+keyLength:], entry) + + return blob[:blobLength] +} + +func readEntry(data []byte) []byte { + length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:]) + + // copy on read + dst := make([]byte, len(data)-int(headersSizeInBytes+length)) + copy(dst, data[headersSizeInBytes+length:]) + + return dst +} + +func readTimestampFromEntry(data []byte) uint64 { + return binary.LittleEndian.Uint64(data) +} + +func readKeyFromEntry(data []byte) string { + length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:]) + + // copy on read + dst := make([]byte, length) + copy(dst, data[headersSizeInBytes:headersSizeInBytes+length]) + + return bytesToString(dst) +} + +func readHashFromEntry(data []byte) uint64 { + return binary.LittleEndian.Uint64(data[timestampSizeInBytes:]) +} + +func resetKeyFromEntry(data []byte) { + binary.LittleEndian.PutUint64(data[timestampSizeInBytes:], 0) +} diff --git a/vendor/github.com/allegro/bigcache/entry_not_found_error.go b/vendor/github.com/allegro/bigcache/entry_not_found_error.go new file mode 100644 index 0000000000..883bdc29eb --- /dev/null +++ b/vendor/github.com/allegro/bigcache/entry_not_found_error.go @@ -0,0 +1,17 @@ +package bigcache + +import "fmt" + +// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key +type EntryNotFoundError struct { + key string +} + +func notFound(key string) error { + return &EntryNotFoundError{key} +} + +// Error returned when entry does not exist. +func (e EntryNotFoundError) Error() string { + return fmt.Sprintf("Entry %q not found", e.key) +} diff --git a/vendor/github.com/allegro/bigcache/fnv.go b/vendor/github.com/allegro/bigcache/fnv.go new file mode 100644 index 0000000000..188c9aa6dc --- /dev/null +++ b/vendor/github.com/allegro/bigcache/fnv.go @@ -0,0 +1,28 @@ +package bigcache + +// newDefaultHasher returns a new 64-bit FNV-1a Hasher which makes no memory allocations. +// Its Sum64 method will lay the value out in big-endian byte order. +// See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function +func newDefaultHasher() Hasher { + return fnv64a{} +} + +type fnv64a struct{} + +const ( + // offset64 FNVa offset basis. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash + offset64 = 14695981039346656037 + // prime64 FNVa prime value. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash + prime64 = 1099511628211 +) + +// Sum64 gets the string and returns its uint64 hash value. +func (f fnv64a) Sum64(key string) uint64 { + var hash uint64 = offset64 + for i := 0; i < len(key); i++ { + hash ^= uint64(key[i]) + hash *= prime64 + } + + return hash +} diff --git a/vendor/github.com/allegro/bigcache/hash.go b/vendor/github.com/allegro/bigcache/hash.go new file mode 100644 index 0000000000..5f8ade774d --- /dev/null +++ b/vendor/github.com/allegro/bigcache/hash.go @@ -0,0 +1,8 @@ +package bigcache + +// Hasher is responsible for generating unsigned, 64 bit hash of provided string. Hasher should minimize collisions +// (generating same hash for different strings) and while performance is also important fast functions are preferable (i.e. +// you can use FarmHash family). +type Hasher interface { + Sum64(string) uint64 +} diff --git a/vendor/github.com/allegro/bigcache/iterator.go b/vendor/github.com/allegro/bigcache/iterator.go new file mode 100644 index 0000000000..70b98d9004 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/iterator.go @@ -0,0 +1,122 @@ +package bigcache + +import "sync" + +type iteratorError string + +func (e iteratorError) Error() string { + return string(e) +} + +// ErrInvalidIteratorState is reported when iterator is in invalid state +const ErrInvalidIteratorState = iteratorError("Iterator is in invalid state. Use SetNext() to move to next position") + +// ErrCannotRetrieveEntry is reported when entry cannot be retrieved from underlying +const ErrCannotRetrieveEntry = iteratorError("Could not retrieve entry from cache") + +var emptyEntryInfo = EntryInfo{} + +// EntryInfo holds informations about entry in the cache +type EntryInfo struct { + timestamp uint64 + hash uint64 + key string + value []byte +} + +// Key returns entry's underlying key +func (e EntryInfo) Key() string { + return e.key +} + +// Hash returns entry's hash value +func (e EntryInfo) Hash() uint64 { + return e.hash +} + +// Timestamp returns entry's timestamp (time of insertion) +func (e EntryInfo) Timestamp() uint64 { + return e.timestamp +} + +// Value returns entry's underlying value +func (e EntryInfo) Value() []byte { + return e.value +} + +// EntryInfoIterator allows to iterate over entries in the cache +type EntryInfoIterator struct { + mutex sync.Mutex + cache *BigCache + currentShard int + currentIndex int + elements []uint32 + elementsCount int + valid bool +} + +// SetNext moves to next element and returns true if it exists. +func (it *EntryInfoIterator) SetNext() bool { + it.mutex.Lock() + + it.valid = false + it.currentIndex++ + + if it.elementsCount > it.currentIndex { + it.valid = true + it.mutex.Unlock() + return true + } + + for i := it.currentShard + 1; i < it.cache.config.Shards; i++ { + it.elements, it.elementsCount = it.cache.shards[i].copyKeys() + + // Non empty shard - stick with it + if it.elementsCount > 0 { + it.currentIndex = 0 + it.currentShard = i + it.valid = true + it.mutex.Unlock() + return true + } + } + it.mutex.Unlock() + return false +} + +func newIterator(cache *BigCache) *EntryInfoIterator { + elements, count := cache.shards[0].copyKeys() + + return &EntryInfoIterator{ + cache: cache, + currentShard: 0, + currentIndex: -1, + elements: elements, + elementsCount: count, + } +} + +// Value returns current value from the iterator +func (it *EntryInfoIterator) Value() (EntryInfo, error) { + it.mutex.Lock() + + if !it.valid { + it.mutex.Unlock() + return emptyEntryInfo, ErrInvalidIteratorState + } + + entry, err := it.cache.shards[it.currentShard].getEntry(int(it.elements[it.currentIndex])) + + if err != nil { + it.mutex.Unlock() + return emptyEntryInfo, ErrCannotRetrieveEntry + } + it.mutex.Unlock() + + return EntryInfo{ + timestamp: readTimestampFromEntry(entry), + hash: readHashFromEntry(entry), + key: readKeyFromEntry(entry), + value: readEntry(entry), + }, nil +} diff --git a/vendor/github.com/allegro/bigcache/logger.go b/vendor/github.com/allegro/bigcache/logger.go new file mode 100644 index 0000000000..50e84abc80 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/logger.go @@ -0,0 +1,30 @@ +package bigcache + +import ( + "log" + "os" +) + +// Logger is invoked when `Config.Verbose=true` +type Logger interface { + Printf(format string, v ...interface{}) +} + +// this is a safeguard, breaking on compile time in case +// `log.Logger` does not adhere to our `Logger` interface. +// see https://golang.org/doc/faq#guarantee_satisfies_interface +var _ Logger = &log.Logger{} + +// DefaultLogger returns a `Logger` implementation +// backed by stdlib's log +func DefaultLogger() *log.Logger { + return log.New(os.Stdout, "", log.LstdFlags) +} + +func newLogger(custom Logger) Logger { + if custom != nil { + return custom + } + + return DefaultLogger() +} diff --git a/vendor/github.com/allegro/bigcache/queue/bytes_queue.go b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go new file mode 100644 index 0000000000..0285c72cd5 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/queue/bytes_queue.go @@ -0,0 +1,210 @@ +package queue + +import ( + "encoding/binary" + "log" + "time" +) + +const ( + // Number of bytes used to keep information about entry size + headerEntrySize = 4 + // Bytes before left margin are not used. Zero index means element does not exist in queue, useful while reading slice from index + leftMarginIndex = 1 + // Minimum empty blob size in bytes. Empty blob fills space between tail and head in additional memory allocation. + // It keeps entries indexes unchanged + minimumEmptyBlobSize = 32 + headerEntrySize +) + +// BytesQueue is a non-thread safe queue type of fifo based on bytes array. +// For every push operation index of entry is returned. It can be used to read the entry later +type BytesQueue struct { + array []byte + capacity int + maxCapacity int + head int + tail int + count int + rightMargin int + headerBuffer []byte + verbose bool + initialCapacity int +} + +type queueError struct { + message string +} + +// NewBytesQueue initialize new bytes queue. +// Initial capacity is used in bytes array allocation +// When verbose flag is set then information about memory allocation are printed +func NewBytesQueue(initialCapacity int, maxCapacity int, verbose bool) *BytesQueue { + return &BytesQueue{ + array: make([]byte, initialCapacity), + capacity: initialCapacity, + maxCapacity: maxCapacity, + headerBuffer: make([]byte, headerEntrySize), + tail: leftMarginIndex, + head: leftMarginIndex, + rightMargin: leftMarginIndex, + verbose: verbose, + initialCapacity: initialCapacity, + } +} + +// Reset removes all entries from queue +func (q *BytesQueue) Reset() { + // Just reset indexes + q.tail = leftMarginIndex + q.head = leftMarginIndex + q.rightMargin = leftMarginIndex + q.count = 0 +} + +// Push copies entry at the end of queue and moves tail pointer. Allocates more space if needed. +// Returns index for pushed data or error if maximum size queue limit is reached. +func (q *BytesQueue) Push(data []byte) (int, error) { + dataLen := len(data) + + if q.availableSpaceAfterTail() < dataLen+headerEntrySize { + if q.availableSpaceBeforeHead() >= dataLen+headerEntrySize { + q.tail = leftMarginIndex + } else if q.capacity+headerEntrySize+dataLen >= q.maxCapacity && q.maxCapacity > 0 { + return -1, &queueError{"Full queue. Maximum size limit reached."} + } else { + q.allocateAdditionalMemory(dataLen + headerEntrySize) + } + } + + index := q.tail + + q.push(data, dataLen) + + return index, nil +} + +func (q *BytesQueue) allocateAdditionalMemory(minimum int) { + start := time.Now() + if q.capacity < minimum { + q.capacity += minimum + } + q.capacity = q.capacity * 2 + if q.capacity > q.maxCapacity && q.maxCapacity > 0 { + q.capacity = q.maxCapacity + } + + oldArray := q.array + q.array = make([]byte, q.capacity) + + if leftMarginIndex != q.rightMargin { + copy(q.array, oldArray[:q.rightMargin]) + + if q.tail < q.head { + emptyBlobLen := q.head - q.tail - headerEntrySize + q.push(make([]byte, emptyBlobLen), emptyBlobLen) + q.head = leftMarginIndex + q.tail = q.rightMargin + } + } + + if q.verbose { + log.Printf("Allocated new queue in %s; Capacity: %d \n", time.Since(start), q.capacity) + } +} + +func (q *BytesQueue) push(data []byte, len int) { + binary.LittleEndian.PutUint32(q.headerBuffer, uint32(len)) + q.copy(q.headerBuffer, headerEntrySize) + + q.copy(data, len) + + if q.tail > q.head { + q.rightMargin = q.tail + } + + q.count++ +} + +func (q *BytesQueue) copy(data []byte, len int) { + q.tail += copy(q.array[q.tail:], data[:len]) +} + +// Pop reads the oldest entry from queue and moves head pointer to the next one +func (q *BytesQueue) Pop() ([]byte, error) { + data, size, err := q.peek(q.head) + if err != nil { + return nil, err + } + + q.head += headerEntrySize + size + q.count-- + + if q.head == q.rightMargin { + q.head = leftMarginIndex + if q.tail == q.rightMargin { + q.tail = leftMarginIndex + } + q.rightMargin = q.tail + } + + return data, nil +} + +// Peek reads the oldest entry from list without moving head pointer +func (q *BytesQueue) Peek() ([]byte, error) { + data, _, err := q.peek(q.head) + return data, err +} + +// Get reads entry from index +func (q *BytesQueue) Get(index int) ([]byte, error) { + data, _, err := q.peek(index) + return data, err +} + +// Capacity returns number of allocated bytes for queue +func (q *BytesQueue) Capacity() int { + return q.capacity +} + +// Len returns number of entries kept in queue +func (q *BytesQueue) Len() int { + return q.count +} + +// Error returns error message +func (e *queueError) Error() string { + return e.message +} + +func (q *BytesQueue) peek(index int) ([]byte, int, error) { + + if q.count == 0 { + return nil, 0, &queueError{"Empty queue"} + } + + if index <= 0 { + return nil, 0, &queueError{"Index must be grater than zero. Invalid index."} + } + + if index+headerEntrySize >= len(q.array) { + return nil, 0, &queueError{"Index out of range"} + } + + blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize])) + return q.array[index+headerEntrySize : index+headerEntrySize+blockSize], blockSize, nil +} + +func (q *BytesQueue) availableSpaceAfterTail() int { + if q.tail >= q.head { + return q.capacity - q.tail + } + return q.head - q.tail - minimumEmptyBlobSize +} + +func (q *BytesQueue) availableSpaceBeforeHead() int { + if q.tail >= q.head { + return q.head - leftMarginIndex - minimumEmptyBlobSize + } + return q.head - q.tail - minimumEmptyBlobSize +} diff --git a/vendor/github.com/allegro/bigcache/shard.go b/vendor/github.com/allegro/bigcache/shard.go new file mode 100644 index 0000000000..56a9fb0895 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/shard.go @@ -0,0 +1,236 @@ +package bigcache + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/allegro/bigcache/queue" +) + +type onRemoveCallback func(wrappedEntry []byte, reason RemoveReason) + +type cacheShard struct { + hashmap map[uint64]uint32 + entries queue.BytesQueue + lock sync.RWMutex + entryBuffer []byte + onRemove onRemoveCallback + + isVerbose bool + logger Logger + clock clock + lifeWindow uint64 + + stats Stats +} + +func (s *cacheShard) get(key string, hashedKey uint64) ([]byte, error) { + s.lock.RLock() + itemIndex := s.hashmap[hashedKey] + + if itemIndex == 0 { + s.lock.RUnlock() + s.miss() + return nil, notFound(key) + } + + wrappedEntry, err := s.entries.Get(int(itemIndex)) + if err != nil { + s.lock.RUnlock() + s.miss() + return nil, err + } + if entryKey := readKeyFromEntry(wrappedEntry); key != entryKey { + if s.isVerbose { + s.logger.Printf("Collision detected. Both %q and %q have the same hash %x", key, entryKey, hashedKey) + } + s.lock.RUnlock() + s.collision() + return nil, notFound(key) + } + s.lock.RUnlock() + s.hit() + return readEntry(wrappedEntry), nil +} + +func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error { + currentTimestamp := uint64(s.clock.epoch()) + + s.lock.Lock() + + if previousIndex := s.hashmap[hashedKey]; previousIndex != 0 { + if previousEntry, err := s.entries.Get(int(previousIndex)); err == nil { + resetKeyFromEntry(previousEntry) + } + } + + if oldestEntry, err := s.entries.Peek(); err == nil { + s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry) + } + + w := wrapEntry(currentTimestamp, hashedKey, key, entry, &s.entryBuffer) + + for { + if index, err := s.entries.Push(w); err == nil { + s.hashmap[hashedKey] = uint32(index) + s.lock.Unlock() + return nil + } + if s.removeOldestEntry(NoSpace) != nil { + s.lock.Unlock() + return fmt.Errorf("entry is bigger than max shard size") + } + } +} + +func (s *cacheShard) del(key string, hashedKey uint64) error { + s.lock.RLock() + itemIndex := s.hashmap[hashedKey] + + if itemIndex == 0 { + s.lock.RUnlock() + s.delmiss() + return notFound(key) + } + + wrappedEntry, err := s.entries.Get(int(itemIndex)) + if err != nil { + s.lock.RUnlock() + s.delmiss() + return err + } + s.lock.RUnlock() + + s.lock.Lock() + { + delete(s.hashmap, hashedKey) + s.onRemove(wrappedEntry, Deleted) + resetKeyFromEntry(wrappedEntry) + } + s.lock.Unlock() + + s.delhit() + return nil +} + +func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool { + oldestTimestamp := readTimestampFromEntry(oldestEntry) + if currentTimestamp-oldestTimestamp > s.lifeWindow { + evict(Expired) + return true + } + return false +} + +func (s *cacheShard) cleanUp(currentTimestamp uint64) { + s.lock.Lock() + for { + if oldestEntry, err := s.entries.Peek(); err != nil { + break + } else if evicted := s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry); !evicted { + break + } + } + s.lock.Unlock() +} + +func (s *cacheShard) getOldestEntry() ([]byte, error) { + return s.entries.Peek() +} + +func (s *cacheShard) getEntry(index int) ([]byte, error) { + return s.entries.Get(index) +} + +func (s *cacheShard) copyKeys() (keys []uint32, next int) { + keys = make([]uint32, len(s.hashmap)) + + s.lock.RLock() + + for _, index := range s.hashmap { + keys[next] = index + next++ + } + + s.lock.RUnlock() + return keys, next +} + +func (s *cacheShard) removeOldestEntry(reason RemoveReason) error { + oldest, err := s.entries.Pop() + if err == nil { + hash := readHashFromEntry(oldest) + delete(s.hashmap, hash) + s.onRemove(oldest, reason) + return nil + } + return err +} + +func (s *cacheShard) reset(config Config) { + s.lock.Lock() + s.hashmap = make(map[uint64]uint32, config.initialShardSize()) + s.entryBuffer = make([]byte, config.MaxEntrySize+headersSizeInBytes) + s.entries.Reset() + s.lock.Unlock() +} + +func (s *cacheShard) len() int { + s.lock.RLock() + res := len(s.hashmap) + s.lock.RUnlock() + return res +} + +func (s *cacheShard) capacity() int { + s.lock.RLock() + res := s.entries.Capacity() + s.lock.RUnlock() + return res +} + +func (s *cacheShard) getStats() Stats { + var stats = Stats{ + Hits: atomic.LoadInt64(&s.stats.Hits), + Misses: atomic.LoadInt64(&s.stats.Misses), + DelHits: atomic.LoadInt64(&s.stats.DelHits), + DelMisses: atomic.LoadInt64(&s.stats.DelMisses), + Collisions: atomic.LoadInt64(&s.stats.Collisions), + } + return stats +} + +func (s *cacheShard) hit() { + atomic.AddInt64(&s.stats.Hits, 1) +} + +func (s *cacheShard) miss() { + atomic.AddInt64(&s.stats.Misses, 1) +} + +func (s *cacheShard) delhit() { + atomic.AddInt64(&s.stats.DelHits, 1) +} + +func (s *cacheShard) delmiss() { + atomic.AddInt64(&s.stats.DelMisses, 1) +} + +func (s *cacheShard) collision() { + atomic.AddInt64(&s.stats.Collisions, 1) +} + +func initNewShard(config Config, callback onRemoveCallback, clock clock) *cacheShard { + return &cacheShard{ + hashmap: make(map[uint64]uint32, config.initialShardSize()), + entries: *queue.NewBytesQueue(config.initialShardSize()*config.MaxEntrySize, config.maximumShardSize(), config.Verbose), + entryBuffer: make([]byte, config.MaxEntrySize+headersSizeInBytes), + onRemove: callback, + + isVerbose: config.Verbose, + logger: newLogger(config.Logger), + clock: clock, + lifeWindow: uint64(config.LifeWindow.Seconds()), + } +} diff --git a/vendor/github.com/allegro/bigcache/stats.go b/vendor/github.com/allegro/bigcache/stats.go new file mode 100644 index 0000000000..07157132a2 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/stats.go @@ -0,0 +1,15 @@ +package bigcache + +// Stats stores cache statistics +type Stats struct { + // Hits is a number of successfully found keys + Hits int64 `json:"hits"` + // Misses is a number of not found keys + Misses int64 `json:"misses"` + // DelHits is a number of successfully deleted keys + DelHits int64 `json:"delete_hits"` + // DelMisses is a number of not deleted keys + DelMisses int64 `json:"delete_misses"` + // Collisions is a number of happened key-collisions + Collisions int64 `json:"collisions"` +} diff --git a/vendor/github.com/allegro/bigcache/utils.go b/vendor/github.com/allegro/bigcache/utils.go new file mode 100644 index 0000000000..ca1df79b93 --- /dev/null +++ b/vendor/github.com/allegro/bigcache/utils.go @@ -0,0 +1,16 @@ +package bigcache + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func convertMBToBytes(value int) int { + return value * 1024 * 1024 +} + +func isPowerOfTwo(number int) bool { + return (number & (number - 1)) == 0 +} diff --git a/vendor/github.com/mattn/go-colorable/README.md b/vendor/github.com/mattn/go-colorable/README.md index e84226a735..56729a92ca 100644 --- a/vendor/github.com/mattn/go-colorable/README.md +++ b/vendor/github.com/mattn/go-colorable/README.md @@ -1,5 +1,10 @@ # go-colorable +[![Godoc Reference](https://godoc.org/github.com/mattn/go-colorable?status.svg)](http://godoc.org/github.com/mattn/go-colorable) +[![Build Status](https://travis-ci.org/mattn/go-colorable.svg?branch=master)](https://travis-ci.org/mattn/go-colorable) +[![Coverage Status](https://coveralls.io/repos/github/mattn/go-colorable/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-colorable?branch=master) +[![Go Report Card](https://goreportcard.com/badge/mattn/go-colorable)](https://goreportcard.com/report/mattn/go-colorable) + Colorable writer for windows. For example, most of logger packages doesn't show colors on windows. (I know we can do it with ansicon. But I don't want.) diff --git a/vendor/github.com/mattn/go-colorable/colorable_others.go b/vendor/github.com/mattn/go-colorable/colorable_others.go index a7fe19a8ca..887f203dc7 100644 --- a/vendor/github.com/mattn/go-colorable/colorable_others.go +++ b/vendor/github.com/mattn/go-colorable/colorable_others.go @@ -1,10 +1,13 @@ // +build !windows +// +build !appengine package colorable import ( "io" "os" + + _ "github.com/mattn/go-isatty" ) // NewColorable return new instance of Writer which handle escape sequence. diff --git a/vendor/github.com/mattn/go-colorable/colorable_windows.go b/vendor/github.com/mattn/go-colorable/colorable_windows.go index 628ad904e5..404e10ca02 100644 --- a/vendor/github.com/mattn/go-colorable/colorable_windows.go +++ b/vendor/github.com/mattn/go-colorable/colorable_windows.go @@ -1,3 +1,6 @@ +// +build windows +// +build !appengine + package colorable import ( @@ -26,6 +29,15 @@ const ( backgroundMask = (backgroundRed | backgroundBlue | backgroundGreen | backgroundIntensity) ) +const ( + genericRead = 0x80000000 + genericWrite = 0x40000000 +) + +const ( + consoleTextmodeBuffer = 0x1 +) + type wchar uint16 type short int16 type dword uint32 @@ -65,14 +77,18 @@ var ( procFillConsoleOutputAttribute = kernel32.NewProc("FillConsoleOutputAttribute") procGetConsoleCursorInfo = kernel32.NewProc("GetConsoleCursorInfo") procSetConsoleCursorInfo = kernel32.NewProc("SetConsoleCursorInfo") + procSetConsoleTitle = kernel32.NewProc("SetConsoleTitleW") + procCreateConsoleScreenBuffer = kernel32.NewProc("CreateConsoleScreenBuffer") ) +// Writer provide colorable Writer to the console type Writer struct { - out io.Writer - handle syscall.Handle - lastbuf bytes.Buffer - oldattr word - oldpos coord + out io.Writer + handle syscall.Handle + althandle syscall.Handle + oldattr word + oldpos coord + rest bytes.Buffer } // NewColorable return new instance of Writer which handle escape sequence from File. @@ -86,9 +102,8 @@ func NewColorable(file *os.File) io.Writer { handle := syscall.Handle(file.Fd()) procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) return &Writer{out: file, handle: handle, oldattr: csbi.attributes, oldpos: coord{0, 0}} - } else { - return file } + return file } // NewColorableStdout return new instance of Writer which handle escape sequence for stdout. @@ -360,20 +375,65 @@ var color256 = map[int]int{ 255: 0xeeeeee, } +// `\033]0;TITLESTR\007` +func doTitleSequence(er *bytes.Reader) error { + var c byte + var err error + + c, err = er.ReadByte() + if err != nil { + return err + } + if c != '0' && c != '2' { + return nil + } + c, err = er.ReadByte() + if err != nil { + return err + } + if c != ';' { + return nil + } + title := make([]byte, 0, 80) + for { + c, err = er.ReadByte() + if err != nil { + return err + } + if c == 0x07 || c == '\n' { + break + } + title = append(title, c) + } + if len(title) > 0 { + title8, err := syscall.UTF16PtrFromString(string(title)) + if err == nil { + procSetConsoleTitle.Call(uintptr(unsafe.Pointer(title8))) + } + } + return nil +} + // Write write data on console func (w *Writer) Write(data []byte) (n int, err error) { var csbi consoleScreenBufferInfo procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) - er := bytes.NewReader(data) + handle := w.handle + + var er *bytes.Reader + if w.rest.Len() > 0 { + var rest bytes.Buffer + w.rest.WriteTo(&rest) + w.rest.Reset() + rest.Write(data) + er = bytes.NewReader(rest.Bytes()) + } else { + er = bytes.NewReader(data) + } var bw [1]byte loop: for { - r1, _, err := procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) - if r1 == 0 { - break loop - } - c1, err := er.ReadByte() if err != nil { break loop @@ -385,155 +445,202 @@ loop: } c2, err := er.ReadByte() if err != nil { - w.lastbuf.WriteByte(c1) break loop } - if c2 != 0x5b { - w.lastbuf.WriteByte(c1) - w.lastbuf.WriteByte(c2) + + switch c2 { + case '>': + continue + case ']': + w.rest.WriteByte(c1) + w.rest.WriteByte(c2) + er.WriteTo(&w.rest) + if bytes.IndexByte(w.rest.Bytes(), 0x07) == -1 { + break loop + } + er = bytes.NewReader(w.rest.Bytes()[2:]) + err := doTitleSequence(er) + if err != nil { + break loop + } + w.rest.Reset() + continue + // https://github.com/mattn/go-colorable/issues/27 + case '7': + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) + w.oldpos = csbi.cursorPosition + continue + case '8': + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&w.oldpos))) + continue + case 0x5b: + // execute part after switch + default: continue } + w.rest.WriteByte(c1) + w.rest.WriteByte(c2) + er.WriteTo(&w.rest) + var buf bytes.Buffer var m byte - for { - c, err := er.ReadByte() - if err != nil { - w.lastbuf.WriteByte(c1) - w.lastbuf.WriteByte(c2) - w.lastbuf.Write(buf.Bytes()) - break loop - } + for i, c := range w.rest.Bytes()[2:] { if ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '@' { m = c + er = bytes.NewReader(w.rest.Bytes()[2+i+1:]) + w.rest.Reset() break } buf.Write([]byte(string(c))) } + if m == 0 { + break loop + } - var csbi consoleScreenBufferInfo switch m { case 'A': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) csbi.cursorPosition.y -= short(n) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'B': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) csbi.cursorPosition.y += short(n) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'C': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) - csbi.cursorPosition.x -= short(n) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) + csbi.cursorPosition.x += short(n) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'D': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - if n, err = strconv.Atoi(buf.String()); err == nil { - var csbi consoleScreenBufferInfo - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) - csbi.cursorPosition.x += short(n) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) + csbi.cursorPosition.x -= short(n) + if csbi.cursorPosition.x < 0 { + csbi.cursorPosition.x = 0 } + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'E': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) csbi.cursorPosition.x = 0 csbi.cursorPosition.y += short(n) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'F': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) csbi.cursorPosition.x = 0 csbi.cursorPosition.y -= short(n) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'G': n, err = strconv.Atoi(buf.String()) if err != nil { continue } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) csbi.cursorPosition.x = short(n - 1) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) - case 'H': - token := strings.Split(buf.String(), ";") - if len(token) != 2 { - continue + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + case 'H', 'f': + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) + if buf.Len() > 0 { + token := strings.Split(buf.String(), ";") + switch len(token) { + case 1: + n1, err := strconv.Atoi(token[0]) + if err != nil { + continue + } + csbi.cursorPosition.y = short(n1 - 1) + case 2: + n1, err := strconv.Atoi(token[0]) + if err != nil { + continue + } + n2, err := strconv.Atoi(token[1]) + if err != nil { + continue + } + csbi.cursorPosition.x = short(n2 - 1) + csbi.cursorPosition.y = short(n1 - 1) + } + } else { + csbi.cursorPosition.y = 0 } - n1, err := strconv.Atoi(token[0]) - if err != nil { - continue - } - n2, err := strconv.Atoi(token[1]) - if err != nil { - continue - } - csbi.cursorPosition.x = short(n2 - 1) - csbi.cursorPosition.y = short(n1 - 1) - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition))) case 'J': - n, err := strconv.Atoi(buf.String()) - if err != nil { - continue + n := 0 + if buf.Len() > 0 { + n, err = strconv.Atoi(buf.String()) + if err != nil { + continue + } } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + var count, written dword var cursor coord + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) switch n { case 0: cursor = coord{x: csbi.cursorPosition.x, y: csbi.cursorPosition.y} + count = dword(csbi.size.x) - dword(csbi.cursorPosition.x) + dword(csbi.size.y-csbi.cursorPosition.y)*dword(csbi.size.x) case 1: cursor = coord{x: csbi.window.left, y: csbi.window.top} + count = dword(csbi.size.x) - dword(csbi.cursorPosition.x) + dword(csbi.window.top-csbi.cursorPosition.y)*dword(csbi.size.x) case 2: cursor = coord{x: csbi.window.left, y: csbi.window.top} + count = dword(csbi.size.x) - dword(csbi.cursorPosition.x) + dword(csbi.size.y-csbi.cursorPosition.y)*dword(csbi.size.x) } - var count, written dword - count = dword(csbi.size.x - csbi.cursorPosition.x + (csbi.size.y-csbi.cursorPosition.y)*csbi.size.x) - procFillConsoleOutputCharacter.Call(uintptr(w.handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) - procFillConsoleOutputAttribute.Call(uintptr(w.handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) + procFillConsoleOutputCharacter.Call(uintptr(handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) + procFillConsoleOutputAttribute.Call(uintptr(handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) case 'K': - n, err := strconv.Atoi(buf.String()) - if err != nil { - continue + n := 0 + if buf.Len() > 0 { + n, err = strconv.Atoi(buf.String()) + if err != nil { + continue + } } - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) var cursor coord + var count, written dword switch n { case 0: cursor = coord{x: csbi.cursorPosition.x, y: csbi.cursorPosition.y} + count = dword(csbi.size.x - csbi.cursorPosition.x) case 1: - cursor = coord{x: csbi.window.left, y: csbi.window.top + csbi.cursorPosition.y} + cursor = coord{x: csbi.window.left, y: csbi.cursorPosition.y} + count = dword(csbi.size.x - csbi.cursorPosition.x) case 2: - cursor = coord{x: csbi.window.left, y: csbi.window.top + csbi.cursorPosition.y} + cursor = coord{x: csbi.window.left, y: csbi.cursorPosition.y} + count = dword(csbi.size.x) } - var count, written dword - count = dword(csbi.size.x - csbi.cursorPosition.x) - procFillConsoleOutputCharacter.Call(uintptr(w.handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) - procFillConsoleOutputAttribute.Call(uintptr(w.handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) + procFillConsoleOutputCharacter.Call(uintptr(handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) + procFillConsoleOutputAttribute.Call(uintptr(handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written))) case 'm': - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) attr := csbi.attributes cs := buf.String() if cs == "" { - procSetConsoleTextAttribute.Call(uintptr(w.handle), uintptr(w.oldattr)) + procSetConsoleTextAttribute.Call(uintptr(handle), uintptr(w.oldattr)) continue } token := strings.Split(cs, ";") @@ -547,7 +654,7 @@ loop: attr |= foregroundIntensity case n == 7: attr = ((attr & foregroundMask) << 4) | ((attr & backgroundMask) >> 4) - case 22 == n || n == 25 || n == 25: + case n == 22 || n == 25: attr |= foregroundIntensity case n == 27: attr = ((attr & foregroundMask) << 4) | ((attr & backgroundMask) >> 4) @@ -572,6 +679,21 @@ loop: attr |= n256foreAttr[n256] i += 2 } + } else if len(token) == 5 && token[i+1] == "2" { + var r, g, b int + r, _ = strconv.Atoi(token[i+2]) + g, _ = strconv.Atoi(token[i+3]) + b, _ = strconv.Atoi(token[i+4]) + i += 4 + if r > 127 { + attr |= foregroundRed + } + if g > 127 { + attr |= foregroundGreen + } + if b > 127 { + attr |= foregroundBlue + } } else { attr = attr & (w.oldattr & backgroundMask) } @@ -599,6 +721,21 @@ loop: attr |= n256backAttr[n256] i += 2 } + } else if len(token) == 5 && token[i+1] == "2" { + var r, g, b int + r, _ = strconv.Atoi(token[i+2]) + g, _ = strconv.Atoi(token[i+3]) + b, _ = strconv.Atoi(token[i+4]) + i += 4 + if r > 127 { + attr |= backgroundRed + } + if g > 127 { + attr |= backgroundGreen + } + if b > 127 { + attr |= backgroundBlue + } } else { attr = attr & (w.oldattr & foregroundMask) } @@ -630,33 +767,56 @@ loop: attr |= backgroundBlue } } - procSetConsoleTextAttribute.Call(uintptr(w.handle), uintptr(attr)) + procSetConsoleTextAttribute.Call(uintptr(handle), uintptr(attr)) } } case 'h': + var ci consoleCursorInfo cs := buf.String() - if cs == "?25" { - var ci consoleCursorInfo - procGetConsoleCursorInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&ci))) + if cs == "5>" { + procGetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) + ci.visible = 0 + procSetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) + } else if cs == "?25" { + procGetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) ci.visible = 1 - procSetConsoleCursorInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&ci))) + procSetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) + } else if cs == "?1049" { + if w.althandle == 0 { + h, _, _ := procCreateConsoleScreenBuffer.Call(uintptr(genericRead|genericWrite), 0, 0, uintptr(consoleTextmodeBuffer), 0, 0) + w.althandle = syscall.Handle(h) + if w.althandle != 0 { + handle = w.althandle + } + } } case 'l': + var ci consoleCursorInfo cs := buf.String() - if cs == "?25" { - var ci consoleCursorInfo - procGetConsoleCursorInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&ci))) + if cs == "5>" { + procGetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) + ci.visible = 1 + procSetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) + } else if cs == "?25" { + procGetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) ci.visible = 0 - procSetConsoleCursorInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&ci))) + procSetConsoleCursorInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&ci))) + } else if cs == "?1049" { + if w.althandle != 0 { + syscall.CloseHandle(w.althandle) + w.althandle = 0 + handle = w.handle + } } case 's': - procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi))) + procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi))) w.oldpos = csbi.cursorPosition case 'u': - procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&w.oldpos))) + procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&w.oldpos))) } } - return len(data) - w.lastbuf.Len(), nil + + return len(data), nil } type consoleColor struct { @@ -700,22 +860,22 @@ func (c consoleColor) backgroundAttr() (attr word) { } var color16 = []consoleColor{ - consoleColor{0x000000, false, false, false, false}, - consoleColor{0x000080, false, false, true, false}, - consoleColor{0x008000, false, true, false, false}, - consoleColor{0x008080, false, true, true, false}, - consoleColor{0x800000, true, false, false, false}, - consoleColor{0x800080, true, false, true, false}, - consoleColor{0x808000, true, true, false, false}, - consoleColor{0xc0c0c0, true, true, true, false}, - consoleColor{0x808080, false, false, false, true}, - consoleColor{0x0000ff, false, false, true, true}, - consoleColor{0x00ff00, false, true, false, true}, - consoleColor{0x00ffff, false, true, true, true}, - consoleColor{0xff0000, true, false, false, true}, - consoleColor{0xff00ff, true, false, true, true}, - consoleColor{0xffff00, true, true, false, true}, - consoleColor{0xffffff, true, true, true, true}, + {0x000000, false, false, false, false}, + {0x000080, false, false, true, false}, + {0x008000, false, true, false, false}, + {0x008080, false, true, true, false}, + {0x800000, true, false, false, false}, + {0x800080, true, false, true, false}, + {0x808000, true, true, false, false}, + {0xc0c0c0, true, true, true, false}, + {0x808080, false, false, false, true}, + {0x0000ff, false, false, true, true}, + {0x00ff00, false, true, false, true}, + {0x00ffff, false, true, true, true}, + {0xff0000, true, false, false, true}, + {0xff00ff, true, false, true, true}, + {0xffff00, true, true, false, true}, + {0xffffff, true, true, true, true}, } type hsv struct { diff --git a/vendor/github.com/mattn/go-colorable/noncolorable.go b/vendor/github.com/mattn/go-colorable/noncolorable.go index ca588c78ac..9721e16f4b 100644 --- a/vendor/github.com/mattn/go-colorable/noncolorable.go +++ b/vendor/github.com/mattn/go-colorable/noncolorable.go @@ -7,8 +7,7 @@ import ( // NonColorable hold writer but remove escape sequence. type NonColorable struct { - out io.Writer - lastbuf bytes.Buffer + out io.Writer } // NewNonColorable return new instance of Writer which remove escape sequence from Writer. @@ -33,12 +32,9 @@ loop: } c2, err := er.ReadByte() if err != nil { - w.lastbuf.WriteByte(c1) break loop } if c2 != 0x5b { - w.lastbuf.WriteByte(c1) - w.lastbuf.WriteByte(c2) continue } @@ -46,9 +42,6 @@ loop: for { c, err := er.ReadByte() if err != nil { - w.lastbuf.WriteByte(c1) - w.lastbuf.WriteByte(c2) - w.lastbuf.Write(buf.Bytes()) break loop } if ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '@' { @@ -57,5 +50,6 @@ loop: buf.Write([]byte(string(c))) } } - return len(data) - w.lastbuf.Len(), nil + + return len(data), nil } diff --git a/vendor/github.com/mattn/go-isatty/README.md b/vendor/github.com/mattn/go-isatty/README.md index 8e4365f45e..1e69004bb0 100644 --- a/vendor/github.com/mattn/go-isatty/README.md +++ b/vendor/github.com/mattn/go-isatty/README.md @@ -1,6 +1,9 @@ # go-isatty -[![Build Status](https://travis-ci.org/mattn/go-isatty.svg?branch=master)](https://travis-ci.org/mattn/go-isatty) [![Coverage Status](https://coveralls.io/repos/github/mattn/go-isatty/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-isatty?branch=master) +[![Godoc Reference](https://godoc.org/github.com/mattn/go-isatty?status.svg)](http://godoc.org/github.com/mattn/go-isatty) +[![Build Status](https://travis-ci.org/mattn/go-isatty.svg?branch=master)](https://travis-ci.org/mattn/go-isatty) +[![Coverage Status](https://coveralls.io/repos/github/mattn/go-isatty/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-isatty?branch=master) +[![Go Report Card](https://goreportcard.com/badge/mattn/go-isatty)](https://goreportcard.com/report/mattn/go-isatty) isatty for golang diff --git a/vendor/github.com/mattn/go-isatty/isatty_appengine.go b/vendor/github.com/mattn/go-isatty/isatty_appengine.go deleted file mode 100644 index 83c588773c..0000000000 --- a/vendor/github.com/mattn/go-isatty/isatty_appengine.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build appengine - -package isatty - -// IsTerminal returns true if the file descriptor is terminal which -// is always false on on appengine classic which is a sandboxed PaaS. -func IsTerminal(fd uintptr) bool { - return false -} diff --git a/vendor/github.com/mattn/go-isatty/isatty_bsd.go b/vendor/github.com/mattn/go-isatty/isatty_bsd.go index 42f2514d13..07e93039db 100644 --- a/vendor/github.com/mattn/go-isatty/isatty_bsd.go +++ b/vendor/github.com/mattn/go-isatty/isatty_bsd.go @@ -16,3 +16,9 @@ func IsTerminal(fd uintptr) bool { _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) return err == 0 } + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_linux.go b/vendor/github.com/mattn/go-isatty/isatty_linux.go index 9d24bac1db..1f4002617a 100644 --- a/vendor/github.com/mattn/go-isatty/isatty_linux.go +++ b/vendor/github.com/mattn/go-isatty/isatty_linux.go @@ -1,5 +1,5 @@ // +build linux -// +build !appengine +// +build !appengine,!ppc64,!ppc64le package isatty @@ -16,3 +16,9 @@ func IsTerminal(fd uintptr) bool { _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) return err == 0 } + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_not_windows.go b/vendor/github.com/mattn/go-isatty/isatty_not_windows.go deleted file mode 100644 index 616832d23e..0000000000 --- a/vendor/github.com/mattn/go-isatty/isatty_not_windows.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !windows appengine - -package isatty - -// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2 -// terminal. This is also always false on this environment. -func IsCygwinTerminal(fd uintptr) bool { - return false -} diff --git a/vendor/github.com/mattn/go-isatty/isatty_solaris.go b/vendor/github.com/mattn/go-isatty/isatty_solaris.go index 1f0c6bf53d..bdd5c79a07 100644 --- a/vendor/github.com/mattn/go-isatty/isatty_solaris.go +++ b/vendor/github.com/mattn/go-isatty/isatty_solaris.go @@ -14,3 +14,9 @@ func IsTerminal(fd uintptr) bool { err := unix.IoctlSetTermio(int(fd), unix.TCGETA, &termio) return err == nil } + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth.go b/vendor/github.com/mattn/go-runewidth/runewidth.go index 2164497ad9..82568a1bb9 100644 --- a/vendor/github.com/mattn/go-runewidth/runewidth.go +++ b/vendor/github.com/mattn/go-runewidth/runewidth.go @@ -1,13 +1,24 @@ package runewidth +import "os" + var ( // EastAsianWidth will be set true if the current locale is CJK - EastAsianWidth = IsEastAsian() + EastAsianWidth bool // DefaultCondition is a condition in current locale DefaultCondition = &Condition{EastAsianWidth} ) +func init() { + env := os.Getenv("RUNEWIDTH_EASTASIAN") + if env == "" { + EastAsianWidth = IsEastAsian() + } else { + EastAsianWidth = env == "1" + } +} + type interval struct { first rune last rune @@ -55,6 +66,7 @@ var private = table{ var nonprint = table{ {0x0000, 0x001F}, {0x007F, 0x009F}, {0x00AD, 0x00AD}, {0x070F, 0x070F}, {0x180B, 0x180E}, {0x200B, 0x200F}, + {0x2028, 0x2029}, {0x202A, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF}, {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFE, 0xFFFF}, } diff --git a/vendor/golang.org/x/tools/imports/fastwalk.go b/vendor/golang.org/x/tools/imports/fastwalk.go deleted file mode 100644 index 157c79225b..0000000000 --- a/vendor/golang.org/x/tools/imports/fastwalk.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// A faster implementation of filepath.Walk. -// -// filepath.Walk's design necessarily calls os.Lstat on each file, -// even if the caller needs less info. And goimports only need to know -// the type of each file. The kernel interface provides the type in -// the Readdir call but the standard library ignored it. -// fastwalk_unix.go contains a fork of the syscall routines. -// -// See golang.org/issue/16399 - -package imports - -import ( - "errors" - "os" - "path/filepath" - "runtime" -) - -// traverseLink is a sentinel error for fastWalk, similar to filepath.SkipDir. -var traverseLink = errors.New("traverse symlink, assuming target is a directory") - -// fastWalk walks the file tree rooted at root, calling walkFn for -// each file or directory in the tree, including root. -// -// If fastWalk returns filepath.SkipDir, the directory is skipped. -// -// Unlike filepath.Walk: -// * file stat calls must be done by the user. -// The only provided metadata is the file type, which does not include -// any permission bits. -// * multiple goroutines stat the filesystem concurrently. The provided -// walkFn must be safe for concurrent use. -// * fastWalk can follow symlinks if walkFn returns the traverseLink -// sentinel error. It is the walkFn's responsibility to prevent -// fastWalk from going into symlink cycles. -func fastWalk(root string, walkFn func(path string, typ os.FileMode) error) error { - // TODO(bradfitz): make numWorkers configurable? We used a - // minimum of 4 to give the kernel more info about multiple - // things we want, in hopes its I/O scheduling can take - // advantage of that. Hopefully most are in cache. Maybe 4 is - // even too low of a minimum. Profile more. - numWorkers := 4 - if n := runtime.NumCPU(); n > numWorkers { - numWorkers = n - } - w := &walker{ - fn: walkFn, - enqueuec: make(chan walkItem, numWorkers), // buffered for performance - workc: make(chan walkItem, numWorkers), // buffered for performance - donec: make(chan struct{}), - - // buffered for correctness & not leaking goroutines: - resc: make(chan error, numWorkers), - } - defer close(w.donec) - // TODO(bradfitz): start the workers as needed? maybe not worth it. - for i := 0; i < numWorkers; i++ { - go w.doWork() - } - todo := []walkItem{{dir: root}} - out := 0 - for { - workc := w.workc - var workItem walkItem - if len(todo) == 0 { - workc = nil - } else { - workItem = todo[len(todo)-1] - } - select { - case workc <- workItem: - todo = todo[:len(todo)-1] - out++ - case it := <-w.enqueuec: - todo = append(todo, it) - case err := <-w.resc: - out-- - if err != nil { - return err - } - if out == 0 && len(todo) == 0 { - // It's safe to quit here, as long as the buffered - // enqueue channel isn't also readable, which might - // happen if the worker sends both another unit of - // work and its result before the other select was - // scheduled and both w.resc and w.enqueuec were - // readable. - select { - case it := <-w.enqueuec: - todo = append(todo, it) - default: - return nil - } - } - } - } -} - -// doWork reads directories as instructed (via workc) and runs the -// user's callback function. -func (w *walker) doWork() { - for { - select { - case <-w.donec: - return - case it := <-w.workc: - w.resc <- w.walk(it.dir, !it.callbackDone) - } - } -} - -type walker struct { - fn func(path string, typ os.FileMode) error - - donec chan struct{} // closed on fastWalk's return - workc chan walkItem // to workers - enqueuec chan walkItem // from workers - resc chan error // from workers -} - -type walkItem struct { - dir string - callbackDone bool // callback already called; don't do it again -} - -func (w *walker) enqueue(it walkItem) { - select { - case w.enqueuec <- it: - case <-w.donec: - } -} - -func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error { - joined := dirName + string(os.PathSeparator) + baseName - if typ == os.ModeDir { - w.enqueue(walkItem{dir: joined}) - return nil - } - - err := w.fn(joined, typ) - if typ == os.ModeSymlink { - if err == traverseLink { - // Set callbackDone so we don't call it twice for both the - // symlink-as-symlink and the symlink-as-directory later: - w.enqueue(walkItem{dir: joined, callbackDone: true}) - return nil - } - if err == filepath.SkipDir { - // Permit SkipDir on symlinks too. - return nil - } - } - return err -} -func (w *walker) walk(root string, runUserCallback bool) error { - if runUserCallback { - err := w.fn(root, os.ModeDir) - if err == filepath.SkipDir { - return nil - } - if err != nil { - return err - } - } - - return readDir(root, w.onDirEnt) -} diff --git a/vendor/golang.org/x/tools/imports/fastwalk_dirent_fileno.go b/vendor/golang.org/x/tools/imports/fastwalk_dirent_fileno.go deleted file mode 100644 index f1fd64949d..0000000000 --- a/vendor/golang.org/x/tools/imports/fastwalk_dirent_fileno.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build freebsd openbsd netbsd - -package imports - -import "syscall" - -func direntInode(dirent *syscall.Dirent) uint64 { - return uint64(dirent.Fileno) -} diff --git a/vendor/golang.org/x/tools/imports/fastwalk_dirent_ino.go b/vendor/golang.org/x/tools/imports/fastwalk_dirent_ino.go deleted file mode 100644 index ee85bc4dd4..0000000000 --- a/vendor/golang.org/x/tools/imports/fastwalk_dirent_ino.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux,!appengine darwin - -package imports - -import "syscall" - -func direntInode(dirent *syscall.Dirent) uint64 { - return uint64(dirent.Ino) -} diff --git a/vendor/golang.org/x/tools/imports/fastwalk_portable.go b/vendor/golang.org/x/tools/imports/fastwalk_portable.go deleted file mode 100644 index 6c2658347d..0000000000 --- a/vendor/golang.org/x/tools/imports/fastwalk_portable.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd - -package imports - -import ( - "io/ioutil" - "os" -) - -// readDir calls fn for each directory entry in dirName. -// It does not descend into directories or follow symlinks. -// If fn returns a non-nil error, readDir returns with that error -// immediately. -func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fis, err := ioutil.ReadDir(dirName) - if err != nil { - return err - } - for _, fi := range fis { - if err := fn(dirName, fi.Name(), fi.Mode()&os.ModeType); err != nil { - return err - } - } - return nil -} diff --git a/vendor/golang.org/x/tools/imports/fastwalk_unix.go b/vendor/golang.org/x/tools/imports/fastwalk_unix.go deleted file mode 100644 index 5854233db9..0000000000 --- a/vendor/golang.org/x/tools/imports/fastwalk_unix.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux,!appengine darwin freebsd openbsd netbsd - -package imports - -import ( - "bytes" - "fmt" - "os" - "syscall" - "unsafe" -) - -const blockSize = 8 << 10 - -// unknownFileMode is a sentinel (and bogus) os.FileMode -// value used to represent a syscall.DT_UNKNOWN Dirent.Type. -const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice - -func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fd, err := syscall.Open(dirName, 0, 0) - if err != nil { - return err - } - defer syscall.Close(fd) - - // The buffer must be at least a block long. - buf := make([]byte, blockSize) // stack-allocated; doesn't escape - bufp := 0 // starting read position in buf - nbuf := 0 // end valid data in buf - for { - if bufp >= nbuf { - bufp = 0 - nbuf, err = syscall.ReadDirent(fd, buf) - if err != nil { - return os.NewSyscallError("readdirent", err) - } - if nbuf <= 0 { - return nil - } - } - consumed, name, typ := parseDirEnt(buf[bufp:nbuf]) - bufp += consumed - if name == "" || name == "." || name == ".." { - continue - } - // Fallback for filesystems (like old XFS) that don't - // support Dirent.Type and have DT_UNKNOWN (0) there - // instead. - if typ == unknownFileMode { - fi, err := os.Lstat(dirName + "/" + name) - if err != nil { - // It got deleted in the meantime. - if os.IsNotExist(err) { - continue - } - return err - } - typ = fi.Mode() & os.ModeType - } - if err := fn(dirName, name, typ); err != nil { - return err - } - } -} - -func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) { - // golang.org/issue/15653 - dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0])) - if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v { - panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v)) - } - if len(buf) < int(dirent.Reclen) { - panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen)) - } - consumed = int(dirent.Reclen) - if direntInode(dirent) == 0 { // File absent in directory. - return - } - switch dirent.Type { - case syscall.DT_REG: - typ = 0 - case syscall.DT_DIR: - typ = os.ModeDir - case syscall.DT_LNK: - typ = os.ModeSymlink - case syscall.DT_BLK: - typ = os.ModeDevice - case syscall.DT_FIFO: - typ = os.ModeNamedPipe - case syscall.DT_SOCK: - typ = os.ModeSocket - case syscall.DT_UNKNOWN: - typ = unknownFileMode - default: - // Skip weird things. - // It's probably a DT_WHT (http://lwn.net/Articles/325369/) - // or something. Revisit if/when this package is moved outside - // of goimports. goimports only cares about regular files, - // symlinks, and directories. - return - } - - nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) - nameLen := bytes.IndexByte(nameBuf[:], 0) - if nameLen < 0 { - panic("failed to find terminating 0 byte in dirent") - } - - // Special cases for common things: - if nameLen == 1 && nameBuf[0] == '.' { - name = "." - } else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' { - name = ".." - } else { - name = string(nameBuf[:nameLen]) - } - return -} diff --git a/vendor/golang.org/x/tools/imports/fix.go b/vendor/golang.org/x/tools/imports/fix.go deleted file mode 100644 index c74bdd2c02..0000000000 --- a/vendor/golang.org/x/tools/imports/fix.go +++ /dev/null @@ -1,978 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package imports - -import ( - "bufio" - "bytes" - "fmt" - "go/ast" - "go/build" - "go/parser" - "go/token" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "sort" - "strings" - "sync" - - "golang.org/x/tools/go/ast/astutil" -) - -// Debug controls verbose logging. -var Debug = false - -var ( - inTests = false // set true by fix_test.go; if false, no need to use testMu - testMu sync.RWMutex // guards globals reset by tests; used only if inTests -) - -// If set, LocalPrefix instructs Process to sort import paths with the given -// prefix into another group after 3rd-party packages. -var LocalPrefix string - -// importToGroup is a list of functions which map from an import path to -// a group number. -var importToGroup = []func(importPath string) (num int, ok bool){ - func(importPath string) (num int, ok bool) { - if LocalPrefix != "" && strings.HasPrefix(importPath, LocalPrefix) { - return 3, true - } - return - }, - func(importPath string) (num int, ok bool) { - if strings.HasPrefix(importPath, "appengine") { - return 2, true - } - return - }, - func(importPath string) (num int, ok bool) { - if strings.Contains(importPath, ".") { - return 1, true - } - return - }, -} - -func importGroup(importPath string) int { - for _, fn := range importToGroup { - if n, ok := fn(importPath); ok { - return n - } - } - return 0 -} - -// packageInfo is a summary of features found in a package. -type packageInfo struct { - Globals map[string]bool // symbol => true -} - -// dirPackageInfo gets information from other files in the package. -func dirPackageInfo(srcDir, filename string) (*packageInfo, error) { - considerTests := strings.HasSuffix(filename, "_test.go") - - // Handle file from stdin - if _, err := os.Stat(filename); err != nil { - if os.IsNotExist(err) { - return &packageInfo{}, nil - } - return nil, err - } - - fileBase := filepath.Base(filename) - packageFileInfos, err := ioutil.ReadDir(srcDir) - if err != nil { - return nil, err - } - - info := &packageInfo{Globals: make(map[string]bool)} - for _, fi := range packageFileInfos { - if fi.Name() == fileBase || !strings.HasSuffix(fi.Name(), ".go") { - continue - } - if !considerTests && strings.HasSuffix(fi.Name(), "_test.go") { - continue - } - - fileSet := token.NewFileSet() - root, err := parser.ParseFile(fileSet, filepath.Join(srcDir, fi.Name()), nil, 0) - if err != nil { - continue - } - - for _, decl := range root.Decls { - genDecl, ok := decl.(*ast.GenDecl) - if !ok { - continue - } - - for _, spec := range genDecl.Specs { - valueSpec, ok := spec.(*ast.ValueSpec) - if !ok { - continue - } - info.Globals[valueSpec.Names[0].Name] = true - } - } - } - return info, nil -} - -func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []string, err error) { - // refs are a set of possible package references currently unsatisfied by imports. - // first key: either base package (e.g. "fmt") or renamed package - // second key: referenced package symbol (e.g. "Println") - refs := make(map[string]map[string]bool) - - // decls are the current package imports. key is base package or renamed package. - decls := make(map[string]*ast.ImportSpec) - - abs, err := filepath.Abs(filename) - if err != nil { - return nil, err - } - srcDir := filepath.Dir(abs) - if Debug { - log.Printf("fixImports(filename=%q), abs=%q, srcDir=%q ...", filename, abs, srcDir) - } - - var packageInfo *packageInfo - var loadedPackageInfo bool - - // collect potential uses of packages. - var visitor visitFn - visitor = visitFn(func(node ast.Node) ast.Visitor { - if node == nil { - return visitor - } - switch v := node.(type) { - case *ast.ImportSpec: - if v.Name != nil { - decls[v.Name.Name] = v - break - } - ipath := strings.Trim(v.Path.Value, `"`) - if ipath == "C" { - break - } - local := importPathToName(ipath, srcDir) - decls[local] = v - case *ast.SelectorExpr: - xident, ok := v.X.(*ast.Ident) - if !ok { - break - } - if xident.Obj != nil { - // if the parser can resolve it, it's not a package ref - break - } - pkgName := xident.Name - if refs[pkgName] == nil { - refs[pkgName] = make(map[string]bool) - } - if !loadedPackageInfo { - loadedPackageInfo = true - packageInfo, _ = dirPackageInfo(srcDir, filename) - } - if decls[pkgName] == nil && (packageInfo == nil || !packageInfo.Globals[pkgName]) { - refs[pkgName][v.Sel.Name] = true - } - } - return visitor - }) - ast.Walk(visitor, f) - - // Nil out any unused ImportSpecs, to be removed in following passes - unusedImport := map[string]string{} - for pkg, is := range decls { - if refs[pkg] == nil && pkg != "_" && pkg != "." { - name := "" - if is.Name != nil { - name = is.Name.Name - } - unusedImport[strings.Trim(is.Path.Value, `"`)] = name - } - } - for ipath, name := range unusedImport { - if ipath == "C" { - // Don't remove cgo stuff. - continue - } - astutil.DeleteNamedImport(fset, f, name, ipath) - } - - for pkgName, symbols := range refs { - if len(symbols) == 0 { - // skip over packages already imported - delete(refs, pkgName) - } - } - - // Search for imports matching potential package references. - searches := 0 - type result struct { - ipath string // import path (if err == nil) - name string // optional name to rename import as - err error - } - results := make(chan result) - for pkgName, symbols := range refs { - go func(pkgName string, symbols map[string]bool) { - ipath, rename, err := findImport(pkgName, symbols, filename) - r := result{ipath: ipath, err: err} - if rename { - r.name = pkgName - } - results <- r - }(pkgName, symbols) - searches++ - } - for i := 0; i < searches; i++ { - result := <-results - if result.err != nil { - return nil, result.err - } - if result.ipath != "" { - if result.name != "" { - astutil.AddNamedImport(fset, f, result.name, result.ipath) - } else { - astutil.AddImport(fset, f, result.ipath) - } - added = append(added, result.ipath) - } - } - - return added, nil -} - -// importPathToName returns the package name for the given import path. -var importPathToName func(importPath, srcDir string) (packageName string) = importPathToNameGoPath - -// importPathToNameBasic assumes the package name is the base of import path. -func importPathToNameBasic(importPath, srcDir string) (packageName string) { - return path.Base(importPath) -} - -// importPathToNameGoPath finds out the actual package name, as declared in its .go files. -// If there's a problem, it falls back to using importPathToNameBasic. -func importPathToNameGoPath(importPath, srcDir string) (packageName string) { - // Fast path for standard library without going to disk. - if pkg, ok := stdImportPackage[importPath]; ok { - return pkg - } - - pkgName, err := importPathToNameGoPathParse(importPath, srcDir) - if Debug { - log.Printf("importPathToNameGoPathParse(%q, srcDir=%q) = %q, %v", importPath, srcDir, pkgName, err) - } - if err == nil { - return pkgName - } - return importPathToNameBasic(importPath, srcDir) -} - -// importPathToNameGoPathParse is a faster version of build.Import if -// the only thing desired is the package name. It uses build.FindOnly -// to find the directory and then only parses one file in the package, -// trusting that the files in the directory are consistent. -func importPathToNameGoPathParse(importPath, srcDir string) (packageName string, err error) { - buildPkg, err := build.Import(importPath, srcDir, build.FindOnly) - if err != nil { - return "", err - } - d, err := os.Open(buildPkg.Dir) - if err != nil { - return "", err - } - names, err := d.Readdirnames(-1) - d.Close() - if err != nil { - return "", err - } - sort.Strings(names) // to have predictable behavior - var lastErr error - var nfile int - for _, name := range names { - if !strings.HasSuffix(name, ".go") { - continue - } - if strings.HasSuffix(name, "_test.go") { - continue - } - nfile++ - fullFile := filepath.Join(buildPkg.Dir, name) - - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, fullFile, nil, parser.PackageClauseOnly) - if err != nil { - lastErr = err - continue - } - pkgName := f.Name.Name - if pkgName == "documentation" { - // Special case from go/build.ImportDir, not - // handled by ctx.MatchFile. - continue - } - if pkgName == "main" { - // Also skip package main, assuming it's a +build ignore generator or example. - // Since you can't import a package main anyway, there's no harm here. - continue - } - return pkgName, nil - } - if lastErr != nil { - return "", lastErr - } - return "", fmt.Errorf("no importable package found in %d Go files", nfile) -} - -var stdImportPackage = map[string]string{} // "net/http" => "http" - -func init() { - // Nothing in the standard library has a package name not - // matching its import base name. - for _, pkg := range stdlib { - if _, ok := stdImportPackage[pkg]; !ok { - stdImportPackage[pkg] = path.Base(pkg) - } - } -} - -// Directory-scanning state. -var ( - // scanGoRootOnce guards calling scanGoRoot (for $GOROOT) - scanGoRootOnce sync.Once - // scanGoPathOnce guards calling scanGoPath (for $GOPATH) - scanGoPathOnce sync.Once - - // populateIgnoreOnce guards calling populateIgnore - populateIgnoreOnce sync.Once - ignoredDirs []os.FileInfo - - dirScanMu sync.RWMutex - dirScan map[string]*pkg // abs dir path => *pkg -) - -type pkg struct { - dir string // absolute file path to pkg directory ("/usr/lib/go/src/net/http") - importPath string // full pkg import path ("net/http", "foo/bar/vendor/a/b") - importPathShort string // vendorless import path ("net/http", "a/b") -} - -// byImportPathShortLength sorts by the short import path length, breaking ties on the -// import string itself. -type byImportPathShortLength []*pkg - -func (s byImportPathShortLength) Len() int { return len(s) } -func (s byImportPathShortLength) Less(i, j int) bool { - vi, vj := s[i].importPathShort, s[j].importPathShort - return len(vi) < len(vj) || (len(vi) == len(vj) && vi < vj) - -} -func (s byImportPathShortLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// gate is a semaphore for limiting concurrency. -type gate chan struct{} - -func (g gate) enter() { g <- struct{}{} } -func (g gate) leave() { <-g } - -var visitedSymlinks struct { - sync.Mutex - m map[string]struct{} -} - -// guarded by populateIgnoreOnce; populates ignoredDirs. -func populateIgnore() { - for _, srcDir := range build.Default.SrcDirs() { - if srcDir == filepath.Join(build.Default.GOROOT, "src") { - continue - } - populateIgnoredDirs(srcDir) - } -} - -// populateIgnoredDirs reads an optional config file at /.goimportsignore -// of relative directories to ignore when scanning for go files. -// The provided path is one of the $GOPATH entries with "src" appended. -func populateIgnoredDirs(path string) { - file := filepath.Join(path, ".goimportsignore") - slurp, err := ioutil.ReadFile(file) - if Debug { - if err != nil { - log.Print(err) - } else { - log.Printf("Read %s", file) - } - } - if err != nil { - return - } - bs := bufio.NewScanner(bytes.NewReader(slurp)) - for bs.Scan() { - line := strings.TrimSpace(bs.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - full := filepath.Join(path, line) - if fi, err := os.Stat(full); err == nil { - ignoredDirs = append(ignoredDirs, fi) - if Debug { - log.Printf("Directory added to ignore list: %s", full) - } - } else if Debug { - log.Printf("Error statting entry in .goimportsignore: %v", err) - } - } -} - -func skipDir(fi os.FileInfo) bool { - for _, ignoredDir := range ignoredDirs { - if os.SameFile(fi, ignoredDir) { - return true - } - } - return false -} - -// shouldTraverse reports whether the symlink fi should, found in dir, -// should be followed. It makes sure symlinks were never visited -// before to avoid symlink loops. -func shouldTraverse(dir string, fi os.FileInfo) bool { - path := filepath.Join(dir, fi.Name()) - target, err := filepath.EvalSymlinks(path) - if err != nil { - if !os.IsNotExist(err) { - fmt.Fprintln(os.Stderr, err) - } - return false - } - ts, err := os.Stat(target) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return false - } - if !ts.IsDir() { - return false - } - if skipDir(ts) { - return false - } - - realParent, err := filepath.EvalSymlinks(dir) - if err != nil { - fmt.Fprint(os.Stderr, err) - return false - } - realPath := filepath.Join(realParent, fi.Name()) - visitedSymlinks.Lock() - defer visitedSymlinks.Unlock() - if visitedSymlinks.m == nil { - visitedSymlinks.m = make(map[string]struct{}) - } - if _, ok := visitedSymlinks.m[realPath]; ok { - return false - } - visitedSymlinks.m[realPath] = struct{}{} - return true -} - -var testHookScanDir = func(dir string) {} - -var scanGoRootDone = make(chan struct{}) // closed when scanGoRoot is done - -func scanGoRoot() { - go func() { - scanGoDirs(true) - close(scanGoRootDone) - }() -} - -func scanGoPath() { scanGoDirs(false) } - -func scanGoDirs(goRoot bool) { - if Debug { - which := "$GOROOT" - if !goRoot { - which = "$GOPATH" - } - log.Printf("scanning " + which) - defer log.Printf("scanned " + which) - } - dirScanMu.Lock() - if dirScan == nil { - dirScan = make(map[string]*pkg) - } - dirScanMu.Unlock() - - for _, srcDir := range build.Default.SrcDirs() { - isGoroot := srcDir == filepath.Join(build.Default.GOROOT, "src") - if isGoroot != goRoot { - continue - } - testHookScanDir(srcDir) - walkFn := func(path string, typ os.FileMode) error { - dir := filepath.Dir(path) - if typ.IsRegular() { - if dir == srcDir { - // Doesn't make sense to have regular files - // directly in your $GOPATH/src or $GOROOT/src. - return nil - } - if !strings.HasSuffix(path, ".go") { - return nil - } - dirScanMu.Lock() - if _, dup := dirScan[dir]; !dup { - importpath := filepath.ToSlash(dir[len(srcDir)+len("/"):]) - dirScan[dir] = &pkg{ - importPath: importpath, - importPathShort: vendorlessImportPath(importpath), - dir: dir, - } - } - dirScanMu.Unlock() - return nil - } - if typ == os.ModeDir { - base := filepath.Base(path) - if base == "" || base[0] == '.' || base[0] == '_' || - base == "testdata" || base == "node_modules" { - return filepath.SkipDir - } - fi, err := os.Lstat(path) - if err == nil && skipDir(fi) { - if Debug { - log.Printf("skipping directory %q under %s", fi.Name(), dir) - } - return filepath.SkipDir - } - return nil - } - if typ == os.ModeSymlink { - base := filepath.Base(path) - if strings.HasPrefix(base, ".#") { - // Emacs noise. - return nil - } - fi, err := os.Lstat(path) - if err != nil { - // Just ignore it. - return nil - } - if shouldTraverse(dir, fi) { - return traverseLink - } - } - return nil - } - if err := fastWalk(srcDir, walkFn); err != nil { - log.Printf("goimports: scanning directory %v: %v", srcDir, err) - } - } -} - -// vendorlessImportPath returns the devendorized version of the provided import path. -// e.g. "foo/bar/vendor/a/b" => "a/b" -func vendorlessImportPath(ipath string) string { - // Devendorize for use in import statement. - if i := strings.LastIndex(ipath, "/vendor/"); i >= 0 { - return ipath[i+len("/vendor/"):] - } - if strings.HasPrefix(ipath, "vendor/") { - return ipath[len("vendor/"):] - } - return ipath -} - -// loadExports returns the set of exported symbols in the package at dir. -// It returns nil on error or if the package name in dir does not match expectPackage. -var loadExports func(expectPackage, dir string) map[string]bool = loadExportsGoPath - -func loadExportsGoPath(expectPackage, dir string) map[string]bool { - if Debug { - log.Printf("loading exports in dir %s (seeking package %s)", dir, expectPackage) - } - exports := make(map[string]bool) - - ctx := build.Default - - // ReadDir is like ioutil.ReadDir, but only returns *.go files - // and filters out _test.go files since they're not relevant - // and only slow things down. - ctx.ReadDir = func(dir string) (notTests []os.FileInfo, err error) { - all, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - notTests = all[:0] - for _, fi := range all { - name := fi.Name() - if strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { - notTests = append(notTests, fi) - } - } - return notTests, nil - } - - files, err := ctx.ReadDir(dir) - if err != nil { - log.Print(err) - return nil - } - - fset := token.NewFileSet() - - for _, fi := range files { - match, err := ctx.MatchFile(dir, fi.Name()) - if err != nil || !match { - continue - } - fullFile := filepath.Join(dir, fi.Name()) - f, err := parser.ParseFile(fset, fullFile, nil, 0) - if err != nil { - if Debug { - log.Printf("Parsing %s: %v", fullFile, err) - } - return nil - } - pkgName := f.Name.Name - if pkgName == "documentation" { - // Special case from go/build.ImportDir, not - // handled by ctx.MatchFile. - continue - } - if pkgName != expectPackage { - if Debug { - log.Printf("scan of dir %v is not expected package %v (actually %v)", dir, expectPackage, pkgName) - } - return nil - } - for name := range f.Scope.Objects { - if ast.IsExported(name) { - exports[name] = true - } - } - } - - if Debug { - exportList := make([]string, 0, len(exports)) - for k := range exports { - exportList = append(exportList, k) - } - sort.Strings(exportList) - log.Printf("loaded exports in dir %v (package %v): %v", dir, expectPackage, strings.Join(exportList, ", ")) - } - return exports -} - -// findImport searches for a package with the given symbols. -// If no package is found, findImport returns ("", false, nil) -// -// This is declared as a variable rather than a function so goimports -// can be easily extended by adding a file with an init function. -// -// The rename value tells goimports whether to use the package name as -// a local qualifier in an import. For example, if findImports("pkg", -// "X") returns ("foo/bar", rename=true), then goimports adds the -// import line: -// import pkg "foo/bar" -// to satisfy uses of pkg.X in the file. -var findImport func(pkgName string, symbols map[string]bool, filename string) (foundPkg string, rename bool, err error) = findImportGoPath - -// findImportGoPath is the normal implementation of findImport. -// (Some companies have their own internally.) -func findImportGoPath(pkgName string, symbols map[string]bool, filename string) (foundPkg string, rename bool, err error) { - if inTests { - testMu.RLock() - defer testMu.RUnlock() - } - - // Fast path for the standard library. - // In the common case we hopefully never have to scan the GOPATH, which can - // be slow with moving disks. - if pkg, rename, ok := findImportStdlib(pkgName, symbols); ok { - return pkg, rename, nil - } - if pkgName == "rand" && symbols["Read"] { - // Special-case rand.Read. - // - // If findImportStdlib didn't find it above, don't go - // searching for it, lest it find and pick math/rand - // in GOROOT (new as of Go 1.6) - // - // crypto/rand is the safer choice. - return "", false, nil - } - - // TODO(sameer): look at the import lines for other Go files in the - // local directory, since the user is likely to import the same packages - // in the current Go file. Return rename=true when the other Go files - // use a renamed package that's also used in the current file. - - // Read all the $GOPATH/src/.goimportsignore files before scanning directories. - populateIgnoreOnce.Do(populateIgnore) - - // Start scanning the $GOROOT asynchronously, then run the - // GOPATH scan synchronously if needed, and then wait for the - // $GOROOT to finish. - // - // TODO(bradfitz): run each $GOPATH entry async. But nobody - // really has more than one anyway, so low priority. - scanGoRootOnce.Do(scanGoRoot) // async - if !fileInDir(filename, build.Default.GOROOT) { - scanGoPathOnce.Do(scanGoPath) // blocking - } - <-scanGoRootDone - - // Find candidate packages, looking only at their directory names first. - var candidates []*pkg - for _, pkg := range dirScan { - if pkgIsCandidate(filename, pkgName, pkg) { - candidates = append(candidates, pkg) - } - } - - // Sort the candidates by their import package length, - // assuming that shorter package names are better than long - // ones. Note that this sorts by the de-vendored name, so - // there's no "penalty" for vendoring. - sort.Sort(byImportPathShortLength(candidates)) - if Debug { - for i, pkg := range candidates { - log.Printf("%s candidate %d/%d: %v", pkgName, i+1, len(candidates), pkg.importPathShort) - } - } - - // Collect exports for packages with matching names. - - done := make(chan struct{}) // closed when we find the answer - defer close(done) - - rescv := make([]chan *pkg, len(candidates)) - for i := range candidates { - rescv[i] = make(chan *pkg) - } - const maxConcurrentPackageImport = 4 - loadExportsSem := make(chan struct{}, maxConcurrentPackageImport) - - go func() { - for i, pkg := range candidates { - select { - case loadExportsSem <- struct{}{}: - select { - case <-done: - default: - } - case <-done: - return - } - pkg := pkg - resc := rescv[i] - go func() { - if inTests { - testMu.RLock() - defer testMu.RUnlock() - } - defer func() { <-loadExportsSem }() - exports := loadExports(pkgName, pkg.dir) - - // If it doesn't have the right - // symbols, send nil to mean no match. - for symbol := range symbols { - if !exports[symbol] { - pkg = nil - break - } - } - select { - case resc <- pkg: - case <-done: - } - }() - } - }() - for _, resc := range rescv { - pkg := <-resc - if pkg == nil { - continue - } - // If the package name in the source doesn't match the import path's base, - // return true so the rewriter adds a name (import foo "github.com/bar/go-foo") - needsRename := path.Base(pkg.importPath) != pkgName - return pkg.importPathShort, needsRename, nil - } - return "", false, nil -} - -// pkgIsCandidate reports whether pkg is a candidate for satisfying the -// finding which package pkgIdent in the file named by filename is trying -// to refer to. -// -// This check is purely lexical and is meant to be as fast as possible -// because it's run over all $GOPATH directories to filter out poor -// candidates in order to limit the CPU and I/O later parsing the -// exports in candidate packages. -// -// filename is the file being formatted. -// pkgIdent is the package being searched for, like "client" (if -// searching for "client.New") -func pkgIsCandidate(filename, pkgIdent string, pkg *pkg) bool { - // Check "internal" and "vendor" visibility: - if !canUse(filename, pkg.dir) { - return false - } - - // Speed optimization to minimize disk I/O: - // the last two components on disk must contain the - // package name somewhere. - // - // This permits mismatch naming like directory - // "go-foo" being package "foo", or "pkg.v3" being "pkg", - // or directory "google.golang.org/api/cloudbilling/v1" - // being package "cloudbilling", but doesn't - // permit a directory "foo" to be package - // "bar", which is strongly discouraged - // anyway. There's no reason goimports needs - // to be slow just to accomodate that. - lastTwo := lastTwoComponents(pkg.importPathShort) - if strings.Contains(lastTwo, pkgIdent) { - return true - } - if hasHyphenOrUpperASCII(lastTwo) && !hasHyphenOrUpperASCII(pkgIdent) { - lastTwo = lowerASCIIAndRemoveHyphen(lastTwo) - if strings.Contains(lastTwo, pkgIdent) { - return true - } - } - - return false -} - -func hasHyphenOrUpperASCII(s string) bool { - for i := 0; i < len(s); i++ { - b := s[i] - if b == '-' || ('A' <= b && b <= 'Z') { - return true - } - } - return false -} - -func lowerASCIIAndRemoveHyphen(s string) (ret string) { - buf := make([]byte, 0, len(s)) - for i := 0; i < len(s); i++ { - b := s[i] - switch { - case b == '-': - continue - case 'A' <= b && b <= 'Z': - buf = append(buf, b+('a'-'A')) - default: - buf = append(buf, b) - } - } - return string(buf) -} - -// canUse reports whether the package in dir is usable from filename, -// respecting the Go "internal" and "vendor" visibility rules. -func canUse(filename, dir string) bool { - // Fast path check, before any allocations. If it doesn't contain vendor - // or internal, it's not tricky: - // Note that this can false-negative on directories like "notinternal", - // but we check it correctly below. This is just a fast path. - if !strings.Contains(dir, "vendor") && !strings.Contains(dir, "internal") { - return true - } - - dirSlash := filepath.ToSlash(dir) - if !strings.Contains(dirSlash, "/vendor/") && !strings.Contains(dirSlash, "/internal/") && !strings.HasSuffix(dirSlash, "/internal") { - return true - } - // Vendor or internal directory only visible from children of parent. - // That means the path from the current directory to the target directory - // can contain ../vendor or ../internal but not ../foo/vendor or ../foo/internal - // or bar/vendor or bar/internal. - // After stripping all the leading ../, the only okay place to see vendor or internal - // is at the very beginning of the path. - absfile, err := filepath.Abs(filename) - if err != nil { - return false - } - absdir, err := filepath.Abs(dir) - if err != nil { - return false - } - rel, err := filepath.Rel(absfile, absdir) - if err != nil { - return false - } - relSlash := filepath.ToSlash(rel) - if i := strings.LastIndex(relSlash, "../"); i >= 0 { - relSlash = relSlash[i+len("../"):] - } - return !strings.Contains(relSlash, "/vendor/") && !strings.Contains(relSlash, "/internal/") && !strings.HasSuffix(relSlash, "/internal") -} - -// lastTwoComponents returns at most the last two path components -// of v, using either / or \ as the path separator. -func lastTwoComponents(v string) string { - nslash := 0 - for i := len(v) - 1; i >= 0; i-- { - if v[i] == '/' || v[i] == '\\' { - nslash++ - if nslash == 2 { - return v[i:] - } - } - } - return v -} - -type visitFn func(node ast.Node) ast.Visitor - -func (fn visitFn) Visit(node ast.Node) ast.Visitor { - return fn(node) -} - -func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath string, rename, ok bool) { - for symbol := range symbols { - key := shortPkg + "." + symbol - path := stdlib[key] - if path == "" { - if key == "rand.Read" { - continue - } - return "", false, false - } - if importPath != "" && importPath != path { - // Ambiguous. Symbols pointed to different things. - return "", false, false - } - importPath = path - } - if importPath == "" && shortPkg == "rand" && symbols["Read"] { - return "crypto/rand", false, true - } - return importPath, false, importPath != "" -} - -// fileInDir reports whether the provided file path looks like -// it's in dir. (without hitting the filesystem) -func fileInDir(file, dir string) bool { - rest := strings.TrimPrefix(file, dir) - if len(rest) == len(file) { - // dir is not a prefix of file. - return false - } - // Check for boundary: either nothing (file == dir), or a slash. - return len(rest) == 0 || rest[0] == '/' || rest[0] == '\\' -} diff --git a/vendor/golang.org/x/tools/imports/imports.go b/vendor/golang.org/x/tools/imports/imports.go deleted file mode 100644 index 67573f497e..0000000000 --- a/vendor/golang.org/x/tools/imports/imports.go +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:generate go run mkstdlib.go - -// Package imports implements a Go pretty-printer (like package "go/format") -// that also adds or removes import statements as necessary. -package imports // import "golang.org/x/tools/imports" - -import ( - "bufio" - "bytes" - "fmt" - "go/ast" - "go/format" - "go/parser" - "go/printer" - "go/token" - "io" - "regexp" - "strconv" - "strings" - - "golang.org/x/tools/go/ast/astutil" -) - -// Options specifies options for processing files. -type Options struct { - Fragment bool // Accept fragment of a source file (no package statement) - AllErrors bool // Report all errors (not just the first 10 on different lines) - - Comments bool // Print comments (true if nil *Options provided) - TabIndent bool // Use tabs for indent (true if nil *Options provided) - TabWidth int // Tab width (8 if nil *Options provided) - - FormatOnly bool // Disable the insertion and deletion of imports -} - -// Process formats and adjusts imports for the provided file. -// If opt is nil the defaults are used. -// -// Note that filename's directory influences which imports can be chosen, -// so it is important that filename be accurate. -// To process data ``as if'' it were in filename, pass the data as a non-nil src. -func Process(filename string, src []byte, opt *Options) ([]byte, error) { - if opt == nil { - opt = &Options{Comments: true, TabIndent: true, TabWidth: 8} - } - - fileSet := token.NewFileSet() - file, adjust, err := parse(fileSet, filename, src, opt) - if err != nil { - return nil, err - } - - if !opt.FormatOnly { - _, err = fixImports(fileSet, file, filename) - if err != nil { - return nil, err - } - } - - sortImports(fileSet, file) - imps := astutil.Imports(fileSet, file) - - var spacesBefore []string // import paths we need spaces before - for _, impSection := range imps { - // Within each block of contiguous imports, see if any - // import lines are in different group numbers. If so, - // we'll need to put a space between them so it's - // compatible with gofmt. - lastGroup := -1 - for _, importSpec := range impSection { - importPath, _ := strconv.Unquote(importSpec.Path.Value) - groupNum := importGroup(importPath) - if groupNum != lastGroup && lastGroup != -1 { - spacesBefore = append(spacesBefore, importPath) - } - lastGroup = groupNum - } - - } - - printerMode := printer.UseSpaces - if opt.TabIndent { - printerMode |= printer.TabIndent - } - printConfig := &printer.Config{Mode: printerMode, Tabwidth: opt.TabWidth} - - var buf bytes.Buffer - err = printConfig.Fprint(&buf, fileSet, file) - if err != nil { - return nil, err - } - out := buf.Bytes() - if adjust != nil { - out = adjust(src, out) - } - if len(spacesBefore) > 0 { - out = addImportSpaces(bytes.NewReader(out), spacesBefore) - } - - out, err = format.Source(out) - if err != nil { - return nil, err - } - return out, nil -} - -// parse parses src, which was read from filename, -// as a Go source file or statement list. -func parse(fset *token.FileSet, filename string, src []byte, opt *Options) (*ast.File, func(orig, src []byte) []byte, error) { - parserMode := parser.Mode(0) - if opt.Comments { - parserMode |= parser.ParseComments - } - if opt.AllErrors { - parserMode |= parser.AllErrors - } - - // Try as whole source file. - file, err := parser.ParseFile(fset, filename, src, parserMode) - if err == nil { - return file, nil, nil - } - // If the error is that the source file didn't begin with a - // package line and we accept fragmented input, fall through to - // try as a source fragment. Stop and return on any other error. - if !opt.Fragment || !strings.Contains(err.Error(), "expected 'package'") { - return nil, nil, err - } - - // If this is a declaration list, make it a source file - // by inserting a package clause. - // Insert using a ;, not a newline, so that the line numbers - // in psrc match the ones in src. - psrc := append([]byte("package main;"), src...) - file, err = parser.ParseFile(fset, filename, psrc, parserMode) - if err == nil { - // If a main function exists, we will assume this is a main - // package and leave the file. - if containsMainFunc(file) { - return file, nil, nil - } - - adjust := func(orig, src []byte) []byte { - // Remove the package clause. - // Gofmt has turned the ; into a \n. - src = src[len("package main\n"):] - return matchSpace(orig, src) - } - return file, adjust, nil - } - // If the error is that the source file didn't begin with a - // declaration, fall through to try as a statement list. - // Stop and return on any other error. - if !strings.Contains(err.Error(), "expected declaration") { - return nil, nil, err - } - - // If this is a statement list, make it a source file - // by inserting a package clause and turning the list - // into a function body. This handles expressions too. - // Insert using a ;, not a newline, so that the line numbers - // in fsrc match the ones in src. - fsrc := append(append([]byte("package p; func _() {"), src...), '}') - file, err = parser.ParseFile(fset, filename, fsrc, parserMode) - if err == nil { - adjust := func(orig, src []byte) []byte { - // Remove the wrapping. - // Gofmt has turned the ; into a \n\n. - src = src[len("package p\n\nfunc _() {"):] - src = src[:len(src)-len("}\n")] - // Gofmt has also indented the function body one level. - // Remove that indent. - src = bytes.Replace(src, []byte("\n\t"), []byte("\n"), -1) - return matchSpace(orig, src) - } - return file, adjust, nil - } - - // Failed, and out of options. - return nil, nil, err -} - -// containsMainFunc checks if a file contains a function declaration with the -// function signature 'func main()' -func containsMainFunc(file *ast.File) bool { - for _, decl := range file.Decls { - if f, ok := decl.(*ast.FuncDecl); ok { - if f.Name.Name != "main" { - continue - } - - if len(f.Type.Params.List) != 0 { - continue - } - - if f.Type.Results != nil && len(f.Type.Results.List) != 0 { - continue - } - - return true - } - } - - return false -} - -func cutSpace(b []byte) (before, middle, after []byte) { - i := 0 - for i < len(b) && (b[i] == ' ' || b[i] == '\t' || b[i] == '\n') { - i++ - } - j := len(b) - for j > 0 && (b[j-1] == ' ' || b[j-1] == '\t' || b[j-1] == '\n') { - j-- - } - if i <= j { - return b[:i], b[i:j], b[j:] - } - return nil, nil, b[j:] -} - -// matchSpace reformats src to use the same space context as orig. -// 1) If orig begins with blank lines, matchSpace inserts them at the beginning of src. -// 2) matchSpace copies the indentation of the first non-blank line in orig -// to every non-blank line in src. -// 3) matchSpace copies the trailing space from orig and uses it in place -// of src's trailing space. -func matchSpace(orig []byte, src []byte) []byte { - before, _, after := cutSpace(orig) - i := bytes.LastIndex(before, []byte{'\n'}) - before, indent := before[:i+1], before[i+1:] - - _, src, _ = cutSpace(src) - - var b bytes.Buffer - b.Write(before) - for len(src) > 0 { - line := src - if i := bytes.IndexByte(line, '\n'); i >= 0 { - line, src = line[:i+1], line[i+1:] - } else { - src = nil - } - if len(line) > 0 && line[0] != '\n' { // not blank - b.Write(indent) - } - b.Write(line) - } - b.Write(after) - return b.Bytes() -} - -var impLine = regexp.MustCompile(`^\s+(?:[\w\.]+\s+)?"(.+)"`) - -func addImportSpaces(r io.Reader, breaks []string) []byte { - var out bytes.Buffer - sc := bufio.NewScanner(r) - inImports := false - done := false - for sc.Scan() { - s := sc.Text() - - if !inImports && !done && strings.HasPrefix(s, "import") { - inImports = true - } - if inImports && (strings.HasPrefix(s, "var") || - strings.HasPrefix(s, "func") || - strings.HasPrefix(s, "const") || - strings.HasPrefix(s, "type")) { - done = true - inImports = false - } - if inImports && len(breaks) > 0 { - if m := impLine.FindStringSubmatch(s); m != nil { - if m[1] == breaks[0] { - out.WriteByte('\n') - breaks = breaks[1:] - } - } - } - - fmt.Fprintln(&out, s) - } - return out.Bytes() -} diff --git a/vendor/golang.org/x/tools/imports/mkindex.go b/vendor/golang.org/x/tools/imports/mkindex.go deleted file mode 100644 index 755e2394f2..0000000000 --- a/vendor/golang.org/x/tools/imports/mkindex.go +++ /dev/null @@ -1,173 +0,0 @@ -// +build ignore - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Command mkindex creates the file "pkgindex.go" containing an index of the Go -// standard library. The file is intended to be built as part of the imports -// package, so that the package may be used in environments where a GOROOT is -// not available (such as App Engine). -package main - -import ( - "bytes" - "fmt" - "go/ast" - "go/build" - "go/format" - "go/parser" - "go/token" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "strings" -) - -var ( - pkgIndex = make(map[string][]pkg) - exports = make(map[string]map[string]bool) -) - -func main() { - // Don't use GOPATH. - ctx := build.Default - ctx.GOPATH = "" - - // Populate pkgIndex global from GOROOT. - for _, path := range ctx.SrcDirs() { - f, err := os.Open(path) - if err != nil { - log.Print(err) - continue - } - children, err := f.Readdir(-1) - f.Close() - if err != nil { - log.Print(err) - continue - } - for _, child := range children { - if child.IsDir() { - loadPkg(path, child.Name()) - } - } - } - // Populate exports global. - for _, ps := range pkgIndex { - for _, p := range ps { - e := loadExports(p.dir) - if e != nil { - exports[p.dir] = e - } - } - } - - // Construct source file. - var buf bytes.Buffer - fmt.Fprint(&buf, pkgIndexHead) - fmt.Fprintf(&buf, "var pkgIndexMaster = %#v\n", pkgIndex) - fmt.Fprintf(&buf, "var exportsMaster = %#v\n", exports) - src := buf.Bytes() - - // Replace main.pkg type name with pkg. - src = bytes.Replace(src, []byte("main.pkg"), []byte("pkg"), -1) - // Replace actual GOROOT with "/go". - src = bytes.Replace(src, []byte(ctx.GOROOT), []byte("/go"), -1) - // Add some line wrapping. - src = bytes.Replace(src, []byte("}, "), []byte("},\n"), -1) - src = bytes.Replace(src, []byte("true, "), []byte("true,\n"), -1) - - var err error - src, err = format.Source(src) - if err != nil { - log.Fatal(err) - } - - // Write out source file. - err = ioutil.WriteFile("pkgindex.go", src, 0644) - if err != nil { - log.Fatal(err) - } -} - -const pkgIndexHead = `package imports - -func init() { - pkgIndexOnce.Do(func() { - pkgIndex.m = pkgIndexMaster - }) - loadExports = func(dir string) map[string]bool { - return exportsMaster[dir] - } -} -` - -type pkg struct { - importpath string // full pkg import path, e.g. "net/http" - dir string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt" -} - -var fset = token.NewFileSet() - -func loadPkg(root, importpath string) { - shortName := path.Base(importpath) - if shortName == "testdata" { - return - } - - dir := filepath.Join(root, importpath) - pkgIndex[shortName] = append(pkgIndex[shortName], pkg{ - importpath: importpath, - dir: dir, - }) - - pkgDir, err := os.Open(dir) - if err != nil { - return - } - children, err := pkgDir.Readdir(-1) - pkgDir.Close() - if err != nil { - return - } - for _, child := range children { - name := child.Name() - if name == "" { - continue - } - if c := name[0]; c == '.' || ('0' <= c && c <= '9') { - continue - } - if child.IsDir() { - loadPkg(root, filepath.Join(importpath, name)) - } - } -} - -func loadExports(dir string) map[string]bool { - exports := make(map[string]bool) - buildPkg, err := build.ImportDir(dir, 0) - if err != nil { - if strings.Contains(err.Error(), "no buildable Go source files in") { - return nil - } - log.Printf("could not import %q: %v", dir, err) - return nil - } - for _, file := range buildPkg.GoFiles { - f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0) - if err != nil { - log.Printf("could not parse %q: %v", file, err) - continue - } - for name := range f.Scope.Objects { - if ast.IsExported(name) { - exports[name] = true - } - } - } - return exports -} diff --git a/vendor/golang.org/x/tools/imports/mkstdlib.go b/vendor/golang.org/x/tools/imports/mkstdlib.go deleted file mode 100644 index 1e559e9f50..0000000000 --- a/vendor/golang.org/x/tools/imports/mkstdlib.go +++ /dev/null @@ -1,103 +0,0 @@ -// +build ignore - -// mkstdlib generates the zstdlib.go file, containing the Go standard -// library API symbols. It's baked into the binary to avoid scanning -// GOPATH in the common case. -package main - -import ( - "bufio" - "bytes" - "fmt" - "go/format" - "io" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "regexp" - "sort" - "strings" -) - -func mustOpen(name string) io.Reader { - f, err := os.Open(name) - if err != nil { - log.Fatal(err) - } - return f -} - -func api(base string) string { - return filepath.Join(os.Getenv("GOROOT"), "api", base) -} - -var sym = regexp.MustCompile(`^pkg (\S+).*?, (?:var|func|type|const) ([A-Z]\w*)`) - -func main() { - var buf bytes.Buffer - outf := func(format string, args ...interface{}) { - fmt.Fprintf(&buf, format, args...) - } - outf("// AUTO-GENERATED BY mkstdlib.go\n\n") - outf("package imports\n") - outf("var stdlib = map[string]string{\n") - f := io.MultiReader( - mustOpen(api("go1.txt")), - mustOpen(api("go1.1.txt")), - mustOpen(api("go1.2.txt")), - mustOpen(api("go1.3.txt")), - mustOpen(api("go1.4.txt")), - mustOpen(api("go1.5.txt")), - mustOpen(api("go1.6.txt")), - mustOpen(api("go1.7.txt")), - ) - sc := bufio.NewScanner(f) - fullImport := map[string]string{} // "zip.NewReader" => "archive/zip" - ambiguous := map[string]bool{} - var keys []string - for sc.Scan() { - l := sc.Text() - has := func(v string) bool { return strings.Contains(l, v) } - if has("struct, ") || has("interface, ") || has(", method (") { - continue - } - if m := sym.FindStringSubmatch(l); m != nil { - full := m[1] - key := path.Base(full) + "." + m[2] - if exist, ok := fullImport[key]; ok { - if exist != full { - ambiguous[key] = true - } - } else { - fullImport[key] = full - keys = append(keys, key) - } - } - } - if err := sc.Err(); err != nil { - log.Fatal(err) - } - sort.Strings(keys) - for _, key := range keys { - if ambiguous[key] { - outf("\t// %q is ambiguous\n", key) - } else { - outf("\t%q: %q,\n", key, fullImport[key]) - } - } - outf("\n") - for _, sym := range [...]string{"Alignof", "ArbitraryType", "Offsetof", "Pointer", "Sizeof"} { - outf("\t%q: %q,\n", "unsafe."+sym, "unsafe") - } - outf("}\n") - fmtbuf, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatal(err) - } - err = ioutil.WriteFile("zstdlib.go", fmtbuf, 0666) - if err != nil { - log.Fatal(err) - } -} diff --git a/vendor/golang.org/x/tools/imports/sortimports.go b/vendor/golang.org/x/tools/imports/sortimports.go deleted file mode 100644 index 653afc5177..0000000000 --- a/vendor/golang.org/x/tools/imports/sortimports.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Hacked up copy of go/ast/import.go - -package imports - -import ( - "go/ast" - "go/token" - "sort" - "strconv" -) - -// sortImports sorts runs of consecutive import lines in import blocks in f. -// It also removes duplicate imports when it is possible to do so without data loss. -func sortImports(fset *token.FileSet, f *ast.File) { - for i, d := range f.Decls { - d, ok := d.(*ast.GenDecl) - if !ok || d.Tok != token.IMPORT { - // Not an import declaration, so we're done. - // Imports are always first. - break - } - - if len(d.Specs) == 0 { - // Empty import block, remove it. - f.Decls = append(f.Decls[:i], f.Decls[i+1:]...) - } - - if !d.Lparen.IsValid() { - // Not a block: sorted by default. - continue - } - - // Identify and sort runs of specs on successive lines. - i := 0 - specs := d.Specs[:0] - for j, s := range d.Specs { - if j > i && fset.Position(s.Pos()).Line > 1+fset.Position(d.Specs[j-1].End()).Line { - // j begins a new run. End this one. - specs = append(specs, sortSpecs(fset, f, d.Specs[i:j])...) - i = j - } - } - specs = append(specs, sortSpecs(fset, f, d.Specs[i:])...) - d.Specs = specs - - // Deduping can leave a blank line before the rparen; clean that up. - if len(d.Specs) > 0 { - lastSpec := d.Specs[len(d.Specs)-1] - lastLine := fset.Position(lastSpec.Pos()).Line - if rParenLine := fset.Position(d.Rparen).Line; rParenLine > lastLine+1 { - fset.File(d.Rparen).MergeLine(rParenLine - 1) - } - } - } -} - -func importPath(s ast.Spec) string { - t, err := strconv.Unquote(s.(*ast.ImportSpec).Path.Value) - if err == nil { - return t - } - return "" -} - -func importName(s ast.Spec) string { - n := s.(*ast.ImportSpec).Name - if n == nil { - return "" - } - return n.Name -} - -func importComment(s ast.Spec) string { - c := s.(*ast.ImportSpec).Comment - if c == nil { - return "" - } - return c.Text() -} - -// collapse indicates whether prev may be removed, leaving only next. -func collapse(prev, next ast.Spec) bool { - if importPath(next) != importPath(prev) || importName(next) != importName(prev) { - return false - } - return prev.(*ast.ImportSpec).Comment == nil -} - -type posSpan struct { - Start token.Pos - End token.Pos -} - -func sortSpecs(fset *token.FileSet, f *ast.File, specs []ast.Spec) []ast.Spec { - // Can't short-circuit here even if specs are already sorted, - // since they might yet need deduplication. - // A lone import, however, may be safely ignored. - if len(specs) <= 1 { - return specs - } - - // Record positions for specs. - pos := make([]posSpan, len(specs)) - for i, s := range specs { - pos[i] = posSpan{s.Pos(), s.End()} - } - - // Identify comments in this range. - // Any comment from pos[0].Start to the final line counts. - lastLine := fset.Position(pos[len(pos)-1].End).Line - cstart := len(f.Comments) - cend := len(f.Comments) - for i, g := range f.Comments { - if g.Pos() < pos[0].Start { - continue - } - if i < cstart { - cstart = i - } - if fset.Position(g.End()).Line > lastLine { - cend = i - break - } - } - comments := f.Comments[cstart:cend] - - // Assign each comment to the import spec preceding it. - importComment := map[*ast.ImportSpec][]*ast.CommentGroup{} - specIndex := 0 - for _, g := range comments { - for specIndex+1 < len(specs) && pos[specIndex+1].Start <= g.Pos() { - specIndex++ - } - s := specs[specIndex].(*ast.ImportSpec) - importComment[s] = append(importComment[s], g) - } - - // Sort the import specs by import path. - // Remove duplicates, when possible without data loss. - // Reassign the import paths to have the same position sequence. - // Reassign each comment to abut the end of its spec. - // Sort the comments by new position. - sort.Sort(byImportSpec(specs)) - - // Dedup. Thanks to our sorting, we can just consider - // adjacent pairs of imports. - deduped := specs[:0] - for i, s := range specs { - if i == len(specs)-1 || !collapse(s, specs[i+1]) { - deduped = append(deduped, s) - } else { - p := s.Pos() - fset.File(p).MergeLine(fset.Position(p).Line) - } - } - specs = deduped - - // Fix up comment positions - for i, s := range specs { - s := s.(*ast.ImportSpec) - if s.Name != nil { - s.Name.NamePos = pos[i].Start - } - s.Path.ValuePos = pos[i].Start - s.EndPos = pos[i].End - for _, g := range importComment[s] { - for _, c := range g.List { - c.Slash = pos[i].End - } - } - } - - sort.Sort(byCommentPos(comments)) - - return specs -} - -type byImportSpec []ast.Spec // slice of *ast.ImportSpec - -func (x byImportSpec) Len() int { return len(x) } -func (x byImportSpec) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x byImportSpec) Less(i, j int) bool { - ipath := importPath(x[i]) - jpath := importPath(x[j]) - - igroup := importGroup(ipath) - jgroup := importGroup(jpath) - if igroup != jgroup { - return igroup < jgroup - } - - if ipath != jpath { - return ipath < jpath - } - iname := importName(x[i]) - jname := importName(x[j]) - - if iname != jname { - return iname < jname - } - return importComment(x[i]) < importComment(x[j]) -} - -type byCommentPos []*ast.CommentGroup - -func (x byCommentPos) Len() int { return len(x) } -func (x byCommentPos) Swap(i, j int) { x[i], x[j] = x[j], x[i] } -func (x byCommentPos) Less(i, j int) bool { return x[i].Pos() < x[j].Pos() } diff --git a/vendor/golang.org/x/tools/imports/zstdlib.go b/vendor/golang.org/x/tools/imports/zstdlib.go deleted file mode 100644 index 28835da031..0000000000 --- a/vendor/golang.org/x/tools/imports/zstdlib.go +++ /dev/null @@ -1,9289 +0,0 @@ -// AUTO-GENERATED BY mkstdlib.go - -package imports - -var stdlib = map[string]string{ - "adler32.Checksum": "hash/adler32", - "adler32.New": "hash/adler32", - "adler32.Size": "hash/adler32", - "aes.BlockSize": "crypto/aes", - "aes.KeySizeError": "crypto/aes", - "aes.NewCipher": "crypto/aes", - "ascii85.CorruptInputError": "encoding/ascii85", - "ascii85.Decode": "encoding/ascii85", - "ascii85.Encode": "encoding/ascii85", - "ascii85.MaxEncodedLen": "encoding/ascii85", - "ascii85.NewDecoder": "encoding/ascii85", - "ascii85.NewEncoder": "encoding/ascii85", - "asn1.BitString": "encoding/asn1", - "asn1.ClassApplication": "encoding/asn1", - "asn1.ClassContextSpecific": "encoding/asn1", - "asn1.ClassPrivate": "encoding/asn1", - "asn1.ClassUniversal": "encoding/asn1", - "asn1.Enumerated": "encoding/asn1", - "asn1.Flag": "encoding/asn1", - "asn1.Marshal": "encoding/asn1", - "asn1.ObjectIdentifier": "encoding/asn1", - "asn1.RawContent": "encoding/asn1", - "asn1.RawValue": "encoding/asn1", - "asn1.StructuralError": "encoding/asn1", - "asn1.SyntaxError": "encoding/asn1", - "asn1.TagBitString": "encoding/asn1", - "asn1.TagBoolean": "encoding/asn1", - "asn1.TagEnum": "encoding/asn1", - "asn1.TagGeneralString": "encoding/asn1", - "asn1.TagGeneralizedTime": "encoding/asn1", - "asn1.TagIA5String": "encoding/asn1", - "asn1.TagInteger": "encoding/asn1", - "asn1.TagOID": "encoding/asn1", - "asn1.TagOctetString": "encoding/asn1", - "asn1.TagPrintableString": "encoding/asn1", - "asn1.TagSequence": "encoding/asn1", - "asn1.TagSet": "encoding/asn1", - "asn1.TagT61String": "encoding/asn1", - "asn1.TagUTCTime": "encoding/asn1", - "asn1.TagUTF8String": "encoding/asn1", - "asn1.Unmarshal": "encoding/asn1", - "asn1.UnmarshalWithParams": "encoding/asn1", - "ast.ArrayType": "go/ast", - "ast.AssignStmt": "go/ast", - "ast.Bad": "go/ast", - "ast.BadDecl": "go/ast", - "ast.BadExpr": "go/ast", - "ast.BadStmt": "go/ast", - "ast.BasicLit": "go/ast", - "ast.BinaryExpr": "go/ast", - "ast.BlockStmt": "go/ast", - "ast.BranchStmt": "go/ast", - "ast.CallExpr": "go/ast", - "ast.CaseClause": "go/ast", - "ast.ChanDir": "go/ast", - "ast.ChanType": "go/ast", - "ast.CommClause": "go/ast", - "ast.Comment": "go/ast", - "ast.CommentGroup": "go/ast", - "ast.CommentMap": "go/ast", - "ast.CompositeLit": "go/ast", - "ast.Con": "go/ast", - "ast.DeclStmt": "go/ast", - "ast.DeferStmt": "go/ast", - "ast.Ellipsis": "go/ast", - "ast.EmptyStmt": "go/ast", - "ast.ExprStmt": "go/ast", - "ast.Field": "go/ast", - "ast.FieldFilter": "go/ast", - "ast.FieldList": "go/ast", - "ast.File": "go/ast", - "ast.FileExports": "go/ast", - "ast.Filter": "go/ast", - "ast.FilterDecl": "go/ast", - "ast.FilterFile": "go/ast", - "ast.FilterFuncDuplicates": "go/ast", - "ast.FilterImportDuplicates": "go/ast", - "ast.FilterPackage": "go/ast", - "ast.FilterUnassociatedComments": "go/ast", - "ast.ForStmt": "go/ast", - "ast.Fprint": "go/ast", - "ast.Fun": "go/ast", - "ast.FuncDecl": "go/ast", - "ast.FuncLit": "go/ast", - "ast.FuncType": "go/ast", - "ast.GenDecl": "go/ast", - "ast.GoStmt": "go/ast", - "ast.Ident": "go/ast", - "ast.IfStmt": "go/ast", - "ast.ImportSpec": "go/ast", - "ast.Importer": "go/ast", - "ast.IncDecStmt": "go/ast", - "ast.IndexExpr": "go/ast", - "ast.Inspect": "go/ast", - "ast.InterfaceType": "go/ast", - "ast.IsExported": "go/ast", - "ast.KeyValueExpr": "go/ast", - "ast.LabeledStmt": "go/ast", - "ast.Lbl": "go/ast", - "ast.MapType": "go/ast", - "ast.MergeMode": "go/ast", - "ast.MergePackageFiles": "go/ast", - "ast.NewCommentMap": "go/ast", - "ast.NewIdent": "go/ast", - "ast.NewObj": "go/ast", - "ast.NewPackage": "go/ast", - "ast.NewScope": "go/ast", - "ast.Node": "go/ast", - "ast.NotNilFilter": "go/ast", - "ast.ObjKind": "go/ast", - "ast.Object": "go/ast", - "ast.Package": "go/ast", - "ast.PackageExports": "go/ast", - "ast.ParenExpr": "go/ast", - "ast.Pkg": "go/ast", - "ast.Print": "go/ast", - "ast.RECV": "go/ast", - "ast.RangeStmt": "go/ast", - "ast.ReturnStmt": "go/ast", - "ast.SEND": "go/ast", - "ast.Scope": "go/ast", - "ast.SelectStmt": "go/ast", - "ast.SelectorExpr": "go/ast", - "ast.SendStmt": "go/ast", - "ast.SliceExpr": "go/ast", - "ast.SortImports": "go/ast", - "ast.StarExpr": "go/ast", - "ast.StructType": "go/ast", - "ast.SwitchStmt": "go/ast", - "ast.Typ": "go/ast", - "ast.TypeAssertExpr": "go/ast", - "ast.TypeSpec": "go/ast", - "ast.TypeSwitchStmt": "go/ast", - "ast.UnaryExpr": "go/ast", - "ast.ValueSpec": "go/ast", - "ast.Var": "go/ast", - "ast.Visitor": "go/ast", - "ast.Walk": "go/ast", - "atomic.AddInt32": "sync/atomic", - "atomic.AddInt64": "sync/atomic", - "atomic.AddUint32": "sync/atomic", - "atomic.AddUint64": "sync/atomic", - "atomic.AddUintptr": "sync/atomic", - "atomic.CompareAndSwapInt32": "sync/atomic", - "atomic.CompareAndSwapInt64": "sync/atomic", - "atomic.CompareAndSwapPointer": "sync/atomic", - "atomic.CompareAndSwapUint32": "sync/atomic", - "atomic.CompareAndSwapUint64": "sync/atomic", - "atomic.CompareAndSwapUintptr": "sync/atomic", - "atomic.LoadInt32": "sync/atomic", - "atomic.LoadInt64": "sync/atomic", - "atomic.LoadPointer": "sync/atomic", - "atomic.LoadUint32": "sync/atomic", - "atomic.LoadUint64": "sync/atomic", - "atomic.LoadUintptr": "sync/atomic", - "atomic.StoreInt32": "sync/atomic", - "atomic.StoreInt64": "sync/atomic", - "atomic.StorePointer": "sync/atomic", - "atomic.StoreUint32": "sync/atomic", - "atomic.StoreUint64": "sync/atomic", - "atomic.StoreUintptr": "sync/atomic", - "atomic.SwapInt32": "sync/atomic", - "atomic.SwapInt64": "sync/atomic", - "atomic.SwapPointer": "sync/atomic", - "atomic.SwapUint32": "sync/atomic", - "atomic.SwapUint64": "sync/atomic", - "atomic.SwapUintptr": "sync/atomic", - "atomic.Value": "sync/atomic", - "base32.CorruptInputError": "encoding/base32", - "base32.Encoding": "encoding/base32", - "base32.HexEncoding": "encoding/base32", - "base32.NewDecoder": "encoding/base32", - "base32.NewEncoder": "encoding/base32", - "base32.NewEncoding": "encoding/base32", - "base32.StdEncoding": "encoding/base32", - "base64.CorruptInputError": "encoding/base64", - "base64.Encoding": "encoding/base64", - "base64.NewDecoder": "encoding/base64", - "base64.NewEncoder": "encoding/base64", - "base64.NewEncoding": "encoding/base64", - "base64.NoPadding": "encoding/base64", - "base64.RawStdEncoding": "encoding/base64", - "base64.RawURLEncoding": "encoding/base64", - "base64.StdEncoding": "encoding/base64", - "base64.StdPadding": "encoding/base64", - "base64.URLEncoding": "encoding/base64", - "big.Above": "math/big", - "big.Accuracy": "math/big", - "big.AwayFromZero": "math/big", - "big.Below": "math/big", - "big.ErrNaN": "math/big", - "big.Exact": "math/big", - "big.Float": "math/big", - "big.Int": "math/big", - "big.Jacobi": "math/big", - "big.MaxBase": "math/big", - "big.MaxExp": "math/big", - "big.MaxPrec": "math/big", - "big.MinExp": "math/big", - "big.NewFloat": "math/big", - "big.NewInt": "math/big", - "big.NewRat": "math/big", - "big.ParseFloat": "math/big", - "big.Rat": "math/big", - "big.RoundingMode": "math/big", - "big.ToNearestAway": "math/big", - "big.ToNearestEven": "math/big", - "big.ToNegativeInf": "math/big", - "big.ToPositiveInf": "math/big", - "big.ToZero": "math/big", - "big.Word": "math/big", - "binary.BigEndian": "encoding/binary", - "binary.ByteOrder": "encoding/binary", - "binary.LittleEndian": "encoding/binary", - "binary.MaxVarintLen16": "encoding/binary", - "binary.MaxVarintLen32": "encoding/binary", - "binary.MaxVarintLen64": "encoding/binary", - "binary.PutUvarint": "encoding/binary", - "binary.PutVarint": "encoding/binary", - "binary.Read": "encoding/binary", - "binary.ReadUvarint": "encoding/binary", - "binary.ReadVarint": "encoding/binary", - "binary.Size": "encoding/binary", - "binary.Uvarint": "encoding/binary", - "binary.Varint": "encoding/binary", - "binary.Write": "encoding/binary", - "bufio.ErrAdvanceTooFar": "bufio", - "bufio.ErrBufferFull": "bufio", - "bufio.ErrFinalToken": "bufio", - "bufio.ErrInvalidUnreadByte": "bufio", - "bufio.ErrInvalidUnreadRune": "bufio", - "bufio.ErrNegativeAdvance": "bufio", - "bufio.ErrNegativeCount": "bufio", - "bufio.ErrTooLong": "bufio", - "bufio.MaxScanTokenSize": "bufio", - "bufio.NewReadWriter": "bufio", - "bufio.NewReader": "bufio", - "bufio.NewReaderSize": "bufio", - "bufio.NewScanner": "bufio", - "bufio.NewWriter": "bufio", - "bufio.NewWriterSize": "bufio", - "bufio.ReadWriter": "bufio", - "bufio.Reader": "bufio", - "bufio.ScanBytes": "bufio", - "bufio.ScanLines": "bufio", - "bufio.ScanRunes": "bufio", - "bufio.ScanWords": "bufio", - "bufio.Scanner": "bufio", - "bufio.SplitFunc": "bufio", - "bufio.Writer": "bufio", - "build.AllowBinary": "go/build", - "build.ArchChar": "go/build", - "build.Context": "go/build", - "build.Default": "go/build", - "build.FindOnly": "go/build", - "build.IgnoreVendor": "go/build", - "build.Import": "go/build", - "build.ImportComment": "go/build", - "build.ImportDir": "go/build", - "build.ImportMode": "go/build", - "build.IsLocalImport": "go/build", - "build.MultiplePackageError": "go/build", - "build.NoGoError": "go/build", - "build.Package": "go/build", - "build.ToolDir": "go/build", - "bytes.Buffer": "bytes", - "bytes.Compare": "bytes", - "bytes.Contains": "bytes", - "bytes.ContainsAny": "bytes", - "bytes.ContainsRune": "bytes", - "bytes.Count": "bytes", - "bytes.Equal": "bytes", - "bytes.EqualFold": "bytes", - "bytes.ErrTooLarge": "bytes", - "bytes.Fields": "bytes", - "bytes.FieldsFunc": "bytes", - "bytes.HasPrefix": "bytes", - "bytes.HasSuffix": "bytes", - "bytes.Index": "bytes", - "bytes.IndexAny": "bytes", - "bytes.IndexByte": "bytes", - "bytes.IndexFunc": "bytes", - "bytes.IndexRune": "bytes", - "bytes.Join": "bytes", - "bytes.LastIndex": "bytes", - "bytes.LastIndexAny": "bytes", - "bytes.LastIndexByte": "bytes", - "bytes.LastIndexFunc": "bytes", - "bytes.Map": "bytes", - "bytes.MinRead": "bytes", - "bytes.NewBuffer": "bytes", - "bytes.NewBufferString": "bytes", - "bytes.NewReader": "bytes", - "bytes.Reader": "bytes", - "bytes.Repeat": "bytes", - "bytes.Replace": "bytes", - "bytes.Runes": "bytes", - "bytes.Split": "bytes", - "bytes.SplitAfter": "bytes", - "bytes.SplitAfterN": "bytes", - "bytes.SplitN": "bytes", - "bytes.Title": "bytes", - "bytes.ToLower": "bytes", - "bytes.ToLowerSpecial": "bytes", - "bytes.ToTitle": "bytes", - "bytes.ToTitleSpecial": "bytes", - "bytes.ToUpper": "bytes", - "bytes.ToUpperSpecial": "bytes", - "bytes.Trim": "bytes", - "bytes.TrimFunc": "bytes", - "bytes.TrimLeft": "bytes", - "bytes.TrimLeftFunc": "bytes", - "bytes.TrimPrefix": "bytes", - "bytes.TrimRight": "bytes", - "bytes.TrimRightFunc": "bytes", - "bytes.TrimSpace": "bytes", - "bytes.TrimSuffix": "bytes", - "bzip2.NewReader": "compress/bzip2", - "bzip2.StructuralError": "compress/bzip2", - "cgi.Handler": "net/http/cgi", - "cgi.Request": "net/http/cgi", - "cgi.RequestFromMap": "net/http/cgi", - "cgi.Serve": "net/http/cgi", - "cipher.AEAD": "crypto/cipher", - "cipher.Block": "crypto/cipher", - "cipher.BlockMode": "crypto/cipher", - "cipher.NewCBCDecrypter": "crypto/cipher", - "cipher.NewCBCEncrypter": "crypto/cipher", - "cipher.NewCFBDecrypter": "crypto/cipher", - "cipher.NewCFBEncrypter": "crypto/cipher", - "cipher.NewCTR": "crypto/cipher", - "cipher.NewGCM": "crypto/cipher", - "cipher.NewGCMWithNonceSize": "crypto/cipher", - "cipher.NewOFB": "crypto/cipher", - "cipher.Stream": "crypto/cipher", - "cipher.StreamReader": "crypto/cipher", - "cipher.StreamWriter": "crypto/cipher", - "cmplx.Abs": "math/cmplx", - "cmplx.Acos": "math/cmplx", - "cmplx.Acosh": "math/cmplx", - "cmplx.Asin": "math/cmplx", - "cmplx.Asinh": "math/cmplx", - "cmplx.Atan": "math/cmplx", - "cmplx.Atanh": "math/cmplx", - "cmplx.Conj": "math/cmplx", - "cmplx.Cos": "math/cmplx", - "cmplx.Cosh": "math/cmplx", - "cmplx.Cot": "math/cmplx", - "cmplx.Exp": "math/cmplx", - "cmplx.Inf": "math/cmplx", - "cmplx.IsInf": "math/cmplx", - "cmplx.IsNaN": "math/cmplx", - "cmplx.Log": "math/cmplx", - "cmplx.Log10": "math/cmplx", - "cmplx.NaN": "math/cmplx", - "cmplx.Phase": "math/cmplx", - "cmplx.Polar": "math/cmplx", - "cmplx.Pow": "math/cmplx", - "cmplx.Rect": "math/cmplx", - "cmplx.Sin": "math/cmplx", - "cmplx.Sinh": "math/cmplx", - "cmplx.Sqrt": "math/cmplx", - "cmplx.Tan": "math/cmplx", - "cmplx.Tanh": "math/cmplx", - "color.Alpha": "image/color", - "color.Alpha16": "image/color", - "color.Alpha16Model": "image/color", - "color.AlphaModel": "image/color", - "color.Black": "image/color", - "color.CMYK": "image/color", - "color.CMYKModel": "image/color", - "color.CMYKToRGB": "image/color", - "color.Color": "image/color", - "color.Gray": "image/color", - "color.Gray16": "image/color", - "color.Gray16Model": "image/color", - "color.GrayModel": "image/color", - "color.Model": "image/color", - "color.ModelFunc": "image/color", - "color.NRGBA": "image/color", - "color.NRGBA64": "image/color", - "color.NRGBA64Model": "image/color", - "color.NRGBAModel": "image/color", - "color.NYCbCrA": "image/color", - "color.NYCbCrAModel": "image/color", - "color.Opaque": "image/color", - "color.Palette": "image/color", - "color.RGBA": "image/color", - "color.RGBA64": "image/color", - "color.RGBA64Model": "image/color", - "color.RGBAModel": "image/color", - "color.RGBToCMYK": "image/color", - "color.RGBToYCbCr": "image/color", - "color.Transparent": "image/color", - "color.White": "image/color", - "color.YCbCr": "image/color", - "color.YCbCrModel": "image/color", - "color.YCbCrToRGB": "image/color", - "constant.BinaryOp": "go/constant", - "constant.BitLen": "go/constant", - "constant.Bool": "go/constant", - "constant.BoolVal": "go/constant", - "constant.Bytes": "go/constant", - "constant.Compare": "go/constant", - "constant.Complex": "go/constant", - "constant.Denom": "go/constant", - "constant.Float": "go/constant", - "constant.Float32Val": "go/constant", - "constant.Float64Val": "go/constant", - "constant.Imag": "go/constant", - "constant.Int": "go/constant", - "constant.Int64Val": "go/constant", - "constant.Kind": "go/constant", - "constant.MakeBool": "go/constant", - "constant.MakeFloat64": "go/constant", - "constant.MakeFromBytes": "go/constant", - "constant.MakeFromLiteral": "go/constant", - "constant.MakeImag": "go/constant", - "constant.MakeInt64": "go/constant", - "constant.MakeString": "go/constant", - "constant.MakeUint64": "go/constant", - "constant.MakeUnknown": "go/constant", - "constant.Num": "go/constant", - "constant.Real": "go/constant", - "constant.Shift": "go/constant", - "constant.Sign": "go/constant", - "constant.String": "go/constant", - "constant.StringVal": "go/constant", - "constant.ToComplex": "go/constant", - "constant.ToFloat": "go/constant", - "constant.ToInt": "go/constant", - "constant.Uint64Val": "go/constant", - "constant.UnaryOp": "go/constant", - "constant.Unknown": "go/constant", - "context.Background": "context", - "context.CancelFunc": "context", - "context.Canceled": "context", - "context.Context": "context", - "context.DeadlineExceeded": "context", - "context.TODO": "context", - "context.WithCancel": "context", - "context.WithDeadline": "context", - "context.WithTimeout": "context", - "context.WithValue": "context", - "cookiejar.Jar": "net/http/cookiejar", - "cookiejar.New": "net/http/cookiejar", - "cookiejar.Options": "net/http/cookiejar", - "cookiejar.PublicSuffixList": "net/http/cookiejar", - "crc32.Castagnoli": "hash/crc32", - "crc32.Checksum": "hash/crc32", - "crc32.ChecksumIEEE": "hash/crc32", - "crc32.IEEE": "hash/crc32", - "crc32.IEEETable": "hash/crc32", - "crc32.Koopman": "hash/crc32", - "crc32.MakeTable": "hash/crc32", - "crc32.New": "hash/crc32", - "crc32.NewIEEE": "hash/crc32", - "crc32.Size": "hash/crc32", - "crc32.Table": "hash/crc32", - "crc32.Update": "hash/crc32", - "crc64.Checksum": "hash/crc64", - "crc64.ECMA": "hash/crc64", - "crc64.ISO": "hash/crc64", - "crc64.MakeTable": "hash/crc64", - "crc64.New": "hash/crc64", - "crc64.Size": "hash/crc64", - "crc64.Table": "hash/crc64", - "crc64.Update": "hash/crc64", - "crypto.Decrypter": "crypto", - "crypto.DecrypterOpts": "crypto", - "crypto.Hash": "crypto", - "crypto.MD4": "crypto", - "crypto.MD5": "crypto", - "crypto.MD5SHA1": "crypto", - "crypto.PrivateKey": "crypto", - "crypto.PublicKey": "crypto", - "crypto.RIPEMD160": "crypto", - "crypto.RegisterHash": "crypto", - "crypto.SHA1": "crypto", - "crypto.SHA224": "crypto", - "crypto.SHA256": "crypto", - "crypto.SHA384": "crypto", - "crypto.SHA3_224": "crypto", - "crypto.SHA3_256": "crypto", - "crypto.SHA3_384": "crypto", - "crypto.SHA3_512": "crypto", - "crypto.SHA512": "crypto", - "crypto.SHA512_224": "crypto", - "crypto.SHA512_256": "crypto", - "crypto.Signer": "crypto", - "crypto.SignerOpts": "crypto", - "csv.ErrBareQuote": "encoding/csv", - "csv.ErrFieldCount": "encoding/csv", - "csv.ErrQuote": "encoding/csv", - "csv.ErrTrailingComma": "encoding/csv", - "csv.NewReader": "encoding/csv", - "csv.NewWriter": "encoding/csv", - "csv.ParseError": "encoding/csv", - "csv.Reader": "encoding/csv", - "csv.Writer": "encoding/csv", - "debug.FreeOSMemory": "runtime/debug", - "debug.GCStats": "runtime/debug", - "debug.PrintStack": "runtime/debug", - "debug.ReadGCStats": "runtime/debug", - "debug.SetGCPercent": "runtime/debug", - "debug.SetMaxStack": "runtime/debug", - "debug.SetMaxThreads": "runtime/debug", - "debug.SetPanicOnFault": "runtime/debug", - "debug.SetTraceback": "runtime/debug", - "debug.Stack": "runtime/debug", - "debug.WriteHeapDump": "runtime/debug", - "des.BlockSize": "crypto/des", - "des.KeySizeError": "crypto/des", - "des.NewCipher": "crypto/des", - "des.NewTripleDESCipher": "crypto/des", - "doc.AllDecls": "go/doc", - "doc.AllMethods": "go/doc", - "doc.Example": "go/doc", - "doc.Examples": "go/doc", - "doc.Filter": "go/doc", - "doc.Func": "go/doc", - "doc.IllegalPrefixes": "go/doc", - "doc.Mode": "go/doc", - "doc.New": "go/doc", - "doc.Note": "go/doc", - "doc.Package": "go/doc", - "doc.Synopsis": "go/doc", - "doc.ToHTML": "go/doc", - "doc.ToText": "go/doc", - "doc.Type": "go/doc", - "doc.Value": "go/doc", - "draw.Draw": "image/draw", - "draw.DrawMask": "image/draw", - "draw.Drawer": "image/draw", - "draw.FloydSteinberg": "image/draw", - "draw.Image": "image/draw", - "draw.Op": "image/draw", - "draw.Over": "image/draw", - "draw.Quantizer": "image/draw", - "draw.Src": "image/draw", - "driver.Bool": "database/sql/driver", - "driver.ColumnConverter": "database/sql/driver", - "driver.Conn": "database/sql/driver", - "driver.DefaultParameterConverter": "database/sql/driver", - "driver.Driver": "database/sql/driver", - "driver.ErrBadConn": "database/sql/driver", - "driver.ErrSkip": "database/sql/driver", - "driver.Execer": "database/sql/driver", - "driver.Int32": "database/sql/driver", - "driver.IsScanValue": "database/sql/driver", - "driver.IsValue": "database/sql/driver", - "driver.NotNull": "database/sql/driver", - "driver.Null": "database/sql/driver", - "driver.Queryer": "database/sql/driver", - "driver.Result": "database/sql/driver", - "driver.ResultNoRows": "database/sql/driver", - "driver.Rows": "database/sql/driver", - "driver.RowsAffected": "database/sql/driver", - "driver.Stmt": "database/sql/driver", - "driver.String": "database/sql/driver", - "driver.Tx": "database/sql/driver", - "driver.Value": "database/sql/driver", - "driver.ValueConverter": "database/sql/driver", - "driver.Valuer": "database/sql/driver", - "dsa.ErrInvalidPublicKey": "crypto/dsa", - "dsa.GenerateKey": "crypto/dsa", - "dsa.GenerateParameters": "crypto/dsa", - "dsa.L1024N160": "crypto/dsa", - "dsa.L2048N224": "crypto/dsa", - "dsa.L2048N256": "crypto/dsa", - "dsa.L3072N256": "crypto/dsa", - "dsa.ParameterSizes": "crypto/dsa", - "dsa.Parameters": "crypto/dsa", - "dsa.PrivateKey": "crypto/dsa", - "dsa.PublicKey": "crypto/dsa", - "dsa.Sign": "crypto/dsa", - "dsa.Verify": "crypto/dsa", - "dwarf.AddrType": "debug/dwarf", - "dwarf.ArrayType": "debug/dwarf", - "dwarf.Attr": "debug/dwarf", - "dwarf.AttrAbstractOrigin": "debug/dwarf", - "dwarf.AttrAccessibility": "debug/dwarf", - "dwarf.AttrAddrClass": "debug/dwarf", - "dwarf.AttrAllocated": "debug/dwarf", - "dwarf.AttrArtificial": "debug/dwarf", - "dwarf.AttrAssociated": "debug/dwarf", - "dwarf.AttrBaseTypes": "debug/dwarf", - "dwarf.AttrBitOffset": "debug/dwarf", - "dwarf.AttrBitSize": "debug/dwarf", - "dwarf.AttrByteSize": "debug/dwarf", - "dwarf.AttrCallColumn": "debug/dwarf", - "dwarf.AttrCallFile": "debug/dwarf", - "dwarf.AttrCallLine": "debug/dwarf", - "dwarf.AttrCalling": "debug/dwarf", - "dwarf.AttrCommonRef": "debug/dwarf", - "dwarf.AttrCompDir": "debug/dwarf", - "dwarf.AttrConstValue": "debug/dwarf", - "dwarf.AttrContainingType": "debug/dwarf", - "dwarf.AttrCount": "debug/dwarf", - "dwarf.AttrDataLocation": "debug/dwarf", - "dwarf.AttrDataMemberLoc": "debug/dwarf", - "dwarf.AttrDeclColumn": "debug/dwarf", - "dwarf.AttrDeclFile": "debug/dwarf", - "dwarf.AttrDeclLine": "debug/dwarf", - "dwarf.AttrDeclaration": "debug/dwarf", - "dwarf.AttrDefaultValue": "debug/dwarf", - "dwarf.AttrDescription": "debug/dwarf", - "dwarf.AttrDiscr": "debug/dwarf", - "dwarf.AttrDiscrList": "debug/dwarf", - "dwarf.AttrDiscrValue": "debug/dwarf", - "dwarf.AttrEncoding": "debug/dwarf", - "dwarf.AttrEntrypc": "debug/dwarf", - "dwarf.AttrExtension": "debug/dwarf", - "dwarf.AttrExternal": "debug/dwarf", - "dwarf.AttrFrameBase": "debug/dwarf", - "dwarf.AttrFriend": "debug/dwarf", - "dwarf.AttrHighpc": "debug/dwarf", - "dwarf.AttrIdentifierCase": "debug/dwarf", - "dwarf.AttrImport": "debug/dwarf", - "dwarf.AttrInline": "debug/dwarf", - "dwarf.AttrIsOptional": "debug/dwarf", - "dwarf.AttrLanguage": "debug/dwarf", - "dwarf.AttrLocation": "debug/dwarf", - "dwarf.AttrLowerBound": "debug/dwarf", - "dwarf.AttrLowpc": "debug/dwarf", - "dwarf.AttrMacroInfo": "debug/dwarf", - "dwarf.AttrName": "debug/dwarf", - "dwarf.AttrNamelistItem": "debug/dwarf", - "dwarf.AttrOrdering": "debug/dwarf", - "dwarf.AttrPriority": "debug/dwarf", - "dwarf.AttrProducer": "debug/dwarf", - "dwarf.AttrPrototyped": "debug/dwarf", - "dwarf.AttrRanges": "debug/dwarf", - "dwarf.AttrReturnAddr": "debug/dwarf", - "dwarf.AttrSegment": "debug/dwarf", - "dwarf.AttrSibling": "debug/dwarf", - "dwarf.AttrSpecification": "debug/dwarf", - "dwarf.AttrStartScope": "debug/dwarf", - "dwarf.AttrStaticLink": "debug/dwarf", - "dwarf.AttrStmtList": "debug/dwarf", - "dwarf.AttrStride": "debug/dwarf", - "dwarf.AttrStrideSize": "debug/dwarf", - "dwarf.AttrStringLength": "debug/dwarf", - "dwarf.AttrTrampoline": "debug/dwarf", - "dwarf.AttrType": "debug/dwarf", - "dwarf.AttrUpperBound": "debug/dwarf", - "dwarf.AttrUseLocation": "debug/dwarf", - "dwarf.AttrUseUTF8": "debug/dwarf", - "dwarf.AttrVarParam": "debug/dwarf", - "dwarf.AttrVirtuality": "debug/dwarf", - "dwarf.AttrVisibility": "debug/dwarf", - "dwarf.AttrVtableElemLoc": "debug/dwarf", - "dwarf.BasicType": "debug/dwarf", - "dwarf.BoolType": "debug/dwarf", - "dwarf.CharType": "debug/dwarf", - "dwarf.Class": "debug/dwarf", - "dwarf.ClassAddress": "debug/dwarf", - "dwarf.ClassBlock": "debug/dwarf", - "dwarf.ClassConstant": "debug/dwarf", - "dwarf.ClassExprLoc": "debug/dwarf", - "dwarf.ClassFlag": "debug/dwarf", - "dwarf.ClassLinePtr": "debug/dwarf", - "dwarf.ClassLocListPtr": "debug/dwarf", - "dwarf.ClassMacPtr": "debug/dwarf", - "dwarf.ClassRangeListPtr": "debug/dwarf", - "dwarf.ClassReference": "debug/dwarf", - "dwarf.ClassReferenceAlt": "debug/dwarf", - "dwarf.ClassReferenceSig": "debug/dwarf", - "dwarf.ClassString": "debug/dwarf", - "dwarf.ClassStringAlt": "debug/dwarf", - "dwarf.ClassUnknown": "debug/dwarf", - "dwarf.CommonType": "debug/dwarf", - "dwarf.ComplexType": "debug/dwarf", - "dwarf.Data": "debug/dwarf", - "dwarf.DecodeError": "debug/dwarf", - "dwarf.DotDotDotType": "debug/dwarf", - "dwarf.Entry": "debug/dwarf", - "dwarf.EnumType": "debug/dwarf", - "dwarf.EnumValue": "debug/dwarf", - "dwarf.ErrUnknownPC": "debug/dwarf", - "dwarf.Field": "debug/dwarf", - "dwarf.FloatType": "debug/dwarf", - "dwarf.FuncType": "debug/dwarf", - "dwarf.IntType": "debug/dwarf", - "dwarf.LineEntry": "debug/dwarf", - "dwarf.LineFile": "debug/dwarf", - "dwarf.LineReader": "debug/dwarf", - "dwarf.LineReaderPos": "debug/dwarf", - "dwarf.New": "debug/dwarf", - "dwarf.Offset": "debug/dwarf", - "dwarf.PtrType": "debug/dwarf", - "dwarf.QualType": "debug/dwarf", - "dwarf.Reader": "debug/dwarf", - "dwarf.StructField": "debug/dwarf", - "dwarf.StructType": "debug/dwarf", - "dwarf.Tag": "debug/dwarf", - "dwarf.TagAccessDeclaration": "debug/dwarf", - "dwarf.TagArrayType": "debug/dwarf", - "dwarf.TagBaseType": "debug/dwarf", - "dwarf.TagCatchDwarfBlock": "debug/dwarf", - "dwarf.TagClassType": "debug/dwarf", - "dwarf.TagCommonDwarfBlock": "debug/dwarf", - "dwarf.TagCommonInclusion": "debug/dwarf", - "dwarf.TagCompileUnit": "debug/dwarf", - "dwarf.TagCondition": "debug/dwarf", - "dwarf.TagConstType": "debug/dwarf", - "dwarf.TagConstant": "debug/dwarf", - "dwarf.TagDwarfProcedure": "debug/dwarf", - "dwarf.TagEntryPoint": "debug/dwarf", - "dwarf.TagEnumerationType": "debug/dwarf", - "dwarf.TagEnumerator": "debug/dwarf", - "dwarf.TagFileType": "debug/dwarf", - "dwarf.TagFormalParameter": "debug/dwarf", - "dwarf.TagFriend": "debug/dwarf", - "dwarf.TagImportedDeclaration": "debug/dwarf", - "dwarf.TagImportedModule": "debug/dwarf", - "dwarf.TagImportedUnit": "debug/dwarf", - "dwarf.TagInheritance": "debug/dwarf", - "dwarf.TagInlinedSubroutine": "debug/dwarf", - "dwarf.TagInterfaceType": "debug/dwarf", - "dwarf.TagLabel": "debug/dwarf", - "dwarf.TagLexDwarfBlock": "debug/dwarf", - "dwarf.TagMember": "debug/dwarf", - "dwarf.TagModule": "debug/dwarf", - "dwarf.TagMutableType": "debug/dwarf", - "dwarf.TagNamelist": "debug/dwarf", - "dwarf.TagNamelistItem": "debug/dwarf", - "dwarf.TagNamespace": "debug/dwarf", - "dwarf.TagPackedType": "debug/dwarf", - "dwarf.TagPartialUnit": "debug/dwarf", - "dwarf.TagPointerType": "debug/dwarf", - "dwarf.TagPtrToMemberType": "debug/dwarf", - "dwarf.TagReferenceType": "debug/dwarf", - "dwarf.TagRestrictType": "debug/dwarf", - "dwarf.TagRvalueReferenceType": "debug/dwarf", - "dwarf.TagSetType": "debug/dwarf", - "dwarf.TagSharedType": "debug/dwarf", - "dwarf.TagStringType": "debug/dwarf", - "dwarf.TagStructType": "debug/dwarf", - "dwarf.TagSubprogram": "debug/dwarf", - "dwarf.TagSubrangeType": "debug/dwarf", - "dwarf.TagSubroutineType": "debug/dwarf", - "dwarf.TagTemplateAlias": "debug/dwarf", - "dwarf.TagTemplateTypeParameter": "debug/dwarf", - "dwarf.TagTemplateValueParameter": "debug/dwarf", - "dwarf.TagThrownType": "debug/dwarf", - "dwarf.TagTryDwarfBlock": "debug/dwarf", - "dwarf.TagTypeUnit": "debug/dwarf", - "dwarf.TagTypedef": "debug/dwarf", - "dwarf.TagUnionType": "debug/dwarf", - "dwarf.TagUnspecifiedParameters": "debug/dwarf", - "dwarf.TagUnspecifiedType": "debug/dwarf", - "dwarf.TagVariable": "debug/dwarf", - "dwarf.TagVariant": "debug/dwarf", - "dwarf.TagVariantPart": "debug/dwarf", - "dwarf.TagVolatileType": "debug/dwarf", - "dwarf.TagWithStmt": "debug/dwarf", - "dwarf.Type": "debug/dwarf", - "dwarf.TypedefType": "debug/dwarf", - "dwarf.UcharType": "debug/dwarf", - "dwarf.UintType": "debug/dwarf", - "dwarf.UnspecifiedType": "debug/dwarf", - "dwarf.VoidType": "debug/dwarf", - "ecdsa.GenerateKey": "crypto/ecdsa", - "ecdsa.PrivateKey": "crypto/ecdsa", - "ecdsa.PublicKey": "crypto/ecdsa", - "ecdsa.Sign": "crypto/ecdsa", - "ecdsa.Verify": "crypto/ecdsa", - "elf.ARM_MAGIC_TRAMP_NUMBER": "debug/elf", - "elf.COMPRESS_HIOS": "debug/elf", - "elf.COMPRESS_HIPROC": "debug/elf", - "elf.COMPRESS_LOOS": "debug/elf", - "elf.COMPRESS_LOPROC": "debug/elf", - "elf.COMPRESS_ZLIB": "debug/elf", - "elf.Chdr32": "debug/elf", - "elf.Chdr64": "debug/elf", - "elf.Class": "debug/elf", - "elf.CompressionType": "debug/elf", - "elf.DF_BIND_NOW": "debug/elf", - "elf.DF_ORIGIN": "debug/elf", - "elf.DF_STATIC_TLS": "debug/elf", - "elf.DF_SYMBOLIC": "debug/elf", - "elf.DF_TEXTREL": "debug/elf", - "elf.DT_BIND_NOW": "debug/elf", - "elf.DT_DEBUG": "debug/elf", - "elf.DT_ENCODING": "debug/elf", - "elf.DT_FINI": "debug/elf", - "elf.DT_FINI_ARRAY": "debug/elf", - "elf.DT_FINI_ARRAYSZ": "debug/elf", - "elf.DT_FLAGS": "debug/elf", - "elf.DT_HASH": "debug/elf", - "elf.DT_HIOS": "debug/elf", - "elf.DT_HIPROC": "debug/elf", - "elf.DT_INIT": "debug/elf", - "elf.DT_INIT_ARRAY": "debug/elf", - "elf.DT_INIT_ARRAYSZ": "debug/elf", - "elf.DT_JMPREL": "debug/elf", - "elf.DT_LOOS": "debug/elf", - "elf.DT_LOPROC": "debug/elf", - "elf.DT_NEEDED": "debug/elf", - "elf.DT_NULL": "debug/elf", - "elf.DT_PLTGOT": "debug/elf", - "elf.DT_PLTREL": "debug/elf", - "elf.DT_PLTRELSZ": "debug/elf", - "elf.DT_PREINIT_ARRAY": "debug/elf", - "elf.DT_PREINIT_ARRAYSZ": "debug/elf", - "elf.DT_REL": "debug/elf", - "elf.DT_RELA": "debug/elf", - "elf.DT_RELAENT": "debug/elf", - "elf.DT_RELASZ": "debug/elf", - "elf.DT_RELENT": "debug/elf", - "elf.DT_RELSZ": "debug/elf", - "elf.DT_RPATH": "debug/elf", - "elf.DT_RUNPATH": "debug/elf", - "elf.DT_SONAME": "debug/elf", - "elf.DT_STRSZ": "debug/elf", - "elf.DT_STRTAB": "debug/elf", - "elf.DT_SYMBOLIC": "debug/elf", - "elf.DT_SYMENT": "debug/elf", - "elf.DT_SYMTAB": "debug/elf", - "elf.DT_TEXTREL": "debug/elf", - "elf.DT_VERNEED": "debug/elf", - "elf.DT_VERNEEDNUM": "debug/elf", - "elf.DT_VERSYM": "debug/elf", - "elf.Data": "debug/elf", - "elf.Dyn32": "debug/elf", - "elf.Dyn64": "debug/elf", - "elf.DynFlag": "debug/elf", - "elf.DynTag": "debug/elf", - "elf.EI_ABIVERSION": "debug/elf", - "elf.EI_CLASS": "debug/elf", - "elf.EI_DATA": "debug/elf", - "elf.EI_NIDENT": "debug/elf", - "elf.EI_OSABI": "debug/elf", - "elf.EI_PAD": "debug/elf", - "elf.EI_VERSION": "debug/elf", - "elf.ELFCLASS32": "debug/elf", - "elf.ELFCLASS64": "debug/elf", - "elf.ELFCLASSNONE": "debug/elf", - "elf.ELFDATA2LSB": "debug/elf", - "elf.ELFDATA2MSB": "debug/elf", - "elf.ELFDATANONE": "debug/elf", - "elf.ELFMAG": "debug/elf", - "elf.ELFOSABI_86OPEN": "debug/elf", - "elf.ELFOSABI_AIX": "debug/elf", - "elf.ELFOSABI_ARM": "debug/elf", - "elf.ELFOSABI_FREEBSD": "debug/elf", - "elf.ELFOSABI_HPUX": "debug/elf", - "elf.ELFOSABI_HURD": "debug/elf", - "elf.ELFOSABI_IRIX": "debug/elf", - "elf.ELFOSABI_LINUX": "debug/elf", - "elf.ELFOSABI_MODESTO": "debug/elf", - "elf.ELFOSABI_NETBSD": "debug/elf", - "elf.ELFOSABI_NONE": "debug/elf", - "elf.ELFOSABI_NSK": "debug/elf", - "elf.ELFOSABI_OPENBSD": "debug/elf", - "elf.ELFOSABI_OPENVMS": "debug/elf", - "elf.ELFOSABI_SOLARIS": "debug/elf", - "elf.ELFOSABI_STANDALONE": "debug/elf", - "elf.ELFOSABI_TRU64": "debug/elf", - "elf.EM_386": "debug/elf", - "elf.EM_486": "debug/elf", - "elf.EM_68HC12": "debug/elf", - "elf.EM_68K": "debug/elf", - "elf.EM_860": "debug/elf", - "elf.EM_88K": "debug/elf", - "elf.EM_960": "debug/elf", - "elf.EM_AARCH64": "debug/elf", - "elf.EM_ALPHA": "debug/elf", - "elf.EM_ALPHA_STD": "debug/elf", - "elf.EM_ARC": "debug/elf", - "elf.EM_ARM": "debug/elf", - "elf.EM_COLDFIRE": "debug/elf", - "elf.EM_FR20": "debug/elf", - "elf.EM_H8S": "debug/elf", - "elf.EM_H8_300": "debug/elf", - "elf.EM_H8_300H": "debug/elf", - "elf.EM_H8_500": "debug/elf", - "elf.EM_IA_64": "debug/elf", - "elf.EM_M32": "debug/elf", - "elf.EM_ME16": "debug/elf", - "elf.EM_MIPS": "debug/elf", - "elf.EM_MIPS_RS3_LE": "debug/elf", - "elf.EM_MIPS_RS4_BE": "debug/elf", - "elf.EM_MIPS_X": "debug/elf", - "elf.EM_MMA": "debug/elf", - "elf.EM_NCPU": "debug/elf", - "elf.EM_NDR1": "debug/elf", - "elf.EM_NONE": "debug/elf", - "elf.EM_PARISC": "debug/elf", - "elf.EM_PCP": "debug/elf", - "elf.EM_PPC": "debug/elf", - "elf.EM_PPC64": "debug/elf", - "elf.EM_RCE": "debug/elf", - "elf.EM_RH32": "debug/elf", - "elf.EM_S370": "debug/elf", - "elf.EM_S390": "debug/elf", - "elf.EM_SH": "debug/elf", - "elf.EM_SPARC": "debug/elf", - "elf.EM_SPARC32PLUS": "debug/elf", - "elf.EM_SPARCV9": "debug/elf", - "elf.EM_ST100": "debug/elf", - "elf.EM_STARCORE": "debug/elf", - "elf.EM_TINYJ": "debug/elf", - "elf.EM_TRICORE": "debug/elf", - "elf.EM_V800": "debug/elf", - "elf.EM_VPP500": "debug/elf", - "elf.EM_X86_64": "debug/elf", - "elf.ET_CORE": "debug/elf", - "elf.ET_DYN": "debug/elf", - "elf.ET_EXEC": "debug/elf", - "elf.ET_HIOS": "debug/elf", - "elf.ET_HIPROC": "debug/elf", - "elf.ET_LOOS": "debug/elf", - "elf.ET_LOPROC": "debug/elf", - "elf.ET_NONE": "debug/elf", - "elf.ET_REL": "debug/elf", - "elf.EV_CURRENT": "debug/elf", - "elf.EV_NONE": "debug/elf", - "elf.ErrNoSymbols": "debug/elf", - "elf.File": "debug/elf", - "elf.FileHeader": "debug/elf", - "elf.FormatError": "debug/elf", - "elf.Header32": "debug/elf", - "elf.Header64": "debug/elf", - "elf.ImportedSymbol": "debug/elf", - "elf.Machine": "debug/elf", - "elf.NT_FPREGSET": "debug/elf", - "elf.NT_PRPSINFO": "debug/elf", - "elf.NT_PRSTATUS": "debug/elf", - "elf.NType": "debug/elf", - "elf.NewFile": "debug/elf", - "elf.OSABI": "debug/elf", - "elf.Open": "debug/elf", - "elf.PF_MASKOS": "debug/elf", - "elf.PF_MASKPROC": "debug/elf", - "elf.PF_R": "debug/elf", - "elf.PF_W": "debug/elf", - "elf.PF_X": "debug/elf", - "elf.PT_DYNAMIC": "debug/elf", - "elf.PT_HIOS": "debug/elf", - "elf.PT_HIPROC": "debug/elf", - "elf.PT_INTERP": "debug/elf", - "elf.PT_LOAD": "debug/elf", - "elf.PT_LOOS": "debug/elf", - "elf.PT_LOPROC": "debug/elf", - "elf.PT_NOTE": "debug/elf", - "elf.PT_NULL": "debug/elf", - "elf.PT_PHDR": "debug/elf", - "elf.PT_SHLIB": "debug/elf", - "elf.PT_TLS": "debug/elf", - "elf.Prog": "debug/elf", - "elf.Prog32": "debug/elf", - "elf.Prog64": "debug/elf", - "elf.ProgFlag": "debug/elf", - "elf.ProgHeader": "debug/elf", - "elf.ProgType": "debug/elf", - "elf.R_386": "debug/elf", - "elf.R_386_32": "debug/elf", - "elf.R_386_COPY": "debug/elf", - "elf.R_386_GLOB_DAT": "debug/elf", - "elf.R_386_GOT32": "debug/elf", - "elf.R_386_GOTOFF": "debug/elf", - "elf.R_386_GOTPC": "debug/elf", - "elf.R_386_JMP_SLOT": "debug/elf", - "elf.R_386_NONE": "debug/elf", - "elf.R_386_PC32": "debug/elf", - "elf.R_386_PLT32": "debug/elf", - "elf.R_386_RELATIVE": "debug/elf", - "elf.R_386_TLS_DTPMOD32": "debug/elf", - "elf.R_386_TLS_DTPOFF32": "debug/elf", - "elf.R_386_TLS_GD": "debug/elf", - "elf.R_386_TLS_GD_32": "debug/elf", - "elf.R_386_TLS_GD_CALL": "debug/elf", - "elf.R_386_TLS_GD_POP": "debug/elf", - "elf.R_386_TLS_GD_PUSH": "debug/elf", - "elf.R_386_TLS_GOTIE": "debug/elf", - "elf.R_386_TLS_IE": "debug/elf", - "elf.R_386_TLS_IE_32": "debug/elf", - "elf.R_386_TLS_LDM": "debug/elf", - "elf.R_386_TLS_LDM_32": "debug/elf", - "elf.R_386_TLS_LDM_CALL": "debug/elf", - "elf.R_386_TLS_LDM_POP": "debug/elf", - "elf.R_386_TLS_LDM_PUSH": "debug/elf", - "elf.R_386_TLS_LDO_32": "debug/elf", - "elf.R_386_TLS_LE": "debug/elf", - "elf.R_386_TLS_LE_32": "debug/elf", - "elf.R_386_TLS_TPOFF": "debug/elf", - "elf.R_386_TLS_TPOFF32": "debug/elf", - "elf.R_390": "debug/elf", - "elf.R_390_12": "debug/elf", - "elf.R_390_16": "debug/elf", - "elf.R_390_20": "debug/elf", - "elf.R_390_32": "debug/elf", - "elf.R_390_64": "debug/elf", - "elf.R_390_8": "debug/elf", - "elf.R_390_COPY": "debug/elf", - "elf.R_390_GLOB_DAT": "debug/elf", - "elf.R_390_GOT12": "debug/elf", - "elf.R_390_GOT16": "debug/elf", - "elf.R_390_GOT20": "debug/elf", - "elf.R_390_GOT32": "debug/elf", - "elf.R_390_GOT64": "debug/elf", - "elf.R_390_GOTENT": "debug/elf", - "elf.R_390_GOTOFF": "debug/elf", - "elf.R_390_GOTOFF16": "debug/elf", - "elf.R_390_GOTOFF64": "debug/elf", - "elf.R_390_GOTPC": "debug/elf", - "elf.R_390_GOTPCDBL": "debug/elf", - "elf.R_390_GOTPLT12": "debug/elf", - "elf.R_390_GOTPLT16": "debug/elf", - "elf.R_390_GOTPLT20": "debug/elf", - "elf.R_390_GOTPLT32": "debug/elf", - "elf.R_390_GOTPLT64": "debug/elf", - "elf.R_390_GOTPLTENT": "debug/elf", - "elf.R_390_GOTPLTOFF16": "debug/elf", - "elf.R_390_GOTPLTOFF32": "debug/elf", - "elf.R_390_GOTPLTOFF64": "debug/elf", - "elf.R_390_JMP_SLOT": "debug/elf", - "elf.R_390_NONE": "debug/elf", - "elf.R_390_PC16": "debug/elf", - "elf.R_390_PC16DBL": "debug/elf", - "elf.R_390_PC32": "debug/elf", - "elf.R_390_PC32DBL": "debug/elf", - "elf.R_390_PC64": "debug/elf", - "elf.R_390_PLT16DBL": "debug/elf", - "elf.R_390_PLT32": "debug/elf", - "elf.R_390_PLT32DBL": "debug/elf", - "elf.R_390_PLT64": "debug/elf", - "elf.R_390_RELATIVE": "debug/elf", - "elf.R_390_TLS_DTPMOD": "debug/elf", - "elf.R_390_TLS_DTPOFF": "debug/elf", - "elf.R_390_TLS_GD32": "debug/elf", - "elf.R_390_TLS_GD64": "debug/elf", - "elf.R_390_TLS_GDCALL": "debug/elf", - "elf.R_390_TLS_GOTIE12": "debug/elf", - "elf.R_390_TLS_GOTIE20": "debug/elf", - "elf.R_390_TLS_GOTIE32": "debug/elf", - "elf.R_390_TLS_GOTIE64": "debug/elf", - "elf.R_390_TLS_IE32": "debug/elf", - "elf.R_390_TLS_IE64": "debug/elf", - "elf.R_390_TLS_IEENT": "debug/elf", - "elf.R_390_TLS_LDCALL": "debug/elf", - "elf.R_390_TLS_LDM32": "debug/elf", - "elf.R_390_TLS_LDM64": "debug/elf", - "elf.R_390_TLS_LDO32": "debug/elf", - "elf.R_390_TLS_LDO64": "debug/elf", - "elf.R_390_TLS_LE32": "debug/elf", - "elf.R_390_TLS_LE64": "debug/elf", - "elf.R_390_TLS_LOAD": "debug/elf", - "elf.R_390_TLS_TPOFF": "debug/elf", - "elf.R_AARCH64": "debug/elf", - "elf.R_AARCH64_ABS16": "debug/elf", - "elf.R_AARCH64_ABS32": "debug/elf", - "elf.R_AARCH64_ABS64": "debug/elf", - "elf.R_AARCH64_ADD_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_ADR_GOT_PAGE": "debug/elf", - "elf.R_AARCH64_ADR_PREL_LO21": "debug/elf", - "elf.R_AARCH64_ADR_PREL_PG_HI21": "debug/elf", - "elf.R_AARCH64_ADR_PREL_PG_HI21_NC": "debug/elf", - "elf.R_AARCH64_CALL26": "debug/elf", - "elf.R_AARCH64_CONDBR19": "debug/elf", - "elf.R_AARCH64_COPY": "debug/elf", - "elf.R_AARCH64_GLOB_DAT": "debug/elf", - "elf.R_AARCH64_GOT_LD_PREL19": "debug/elf", - "elf.R_AARCH64_IRELATIVE": "debug/elf", - "elf.R_AARCH64_JUMP26": "debug/elf", - "elf.R_AARCH64_JUMP_SLOT": "debug/elf", - "elf.R_AARCH64_LD64_GOT_LO12_NC": "debug/elf", - "elf.R_AARCH64_LDST128_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_LDST16_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_LDST32_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_LDST64_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_LDST8_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_LD_PREL_LO19": "debug/elf", - "elf.R_AARCH64_MOVW_SABS_G0": "debug/elf", - "elf.R_AARCH64_MOVW_SABS_G1": "debug/elf", - "elf.R_AARCH64_MOVW_SABS_G2": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G0": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G0_NC": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G1": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G1_NC": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G2": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G2_NC": "debug/elf", - "elf.R_AARCH64_MOVW_UABS_G3": "debug/elf", - "elf.R_AARCH64_NONE": "debug/elf", - "elf.R_AARCH64_NULL": "debug/elf", - "elf.R_AARCH64_P32_ABS16": "debug/elf", - "elf.R_AARCH64_P32_ABS32": "debug/elf", - "elf.R_AARCH64_P32_ADD_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_ADR_GOT_PAGE": "debug/elf", - "elf.R_AARCH64_P32_ADR_PREL_LO21": "debug/elf", - "elf.R_AARCH64_P32_ADR_PREL_PG_HI21": "debug/elf", - "elf.R_AARCH64_P32_CALL26": "debug/elf", - "elf.R_AARCH64_P32_CONDBR19": "debug/elf", - "elf.R_AARCH64_P32_COPY": "debug/elf", - "elf.R_AARCH64_P32_GLOB_DAT": "debug/elf", - "elf.R_AARCH64_P32_GOT_LD_PREL19": "debug/elf", - "elf.R_AARCH64_P32_IRELATIVE": "debug/elf", - "elf.R_AARCH64_P32_JUMP26": "debug/elf", - "elf.R_AARCH64_P32_JUMP_SLOT": "debug/elf", - "elf.R_AARCH64_P32_LD32_GOT_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_LDST128_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_LDST16_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_LDST32_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_LDST64_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_LDST8_ABS_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_LD_PREL_LO19": "debug/elf", - "elf.R_AARCH64_P32_MOVW_SABS_G0": "debug/elf", - "elf.R_AARCH64_P32_MOVW_UABS_G0": "debug/elf", - "elf.R_AARCH64_P32_MOVW_UABS_G0_NC": "debug/elf", - "elf.R_AARCH64_P32_MOVW_UABS_G1": "debug/elf", - "elf.R_AARCH64_P32_PREL16": "debug/elf", - "elf.R_AARCH64_P32_PREL32": "debug/elf", - "elf.R_AARCH64_P32_RELATIVE": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC_ADD_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC_ADR_PAGE21": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC_ADR_PREL21": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC_CALL": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC_LD32_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_TLSDESC_LD_PREL19": "debug/elf", - "elf.R_AARCH64_P32_TLSGD_ADD_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_TLSGD_ADR_PAGE21": "debug/elf", - "elf.R_AARCH64_P32_TLSIE_ADR_GOTTPREL_PAGE21": "debug/elf", - "elf.R_AARCH64_P32_TLSIE_LD32_GOTTPREL_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_TLSIE_LD_GOTTPREL_PREL19": "debug/elf", - "elf.R_AARCH64_P32_TLSLE_ADD_TPREL_HI12": "debug/elf", - "elf.R_AARCH64_P32_TLSLE_ADD_TPREL_LO12": "debug/elf", - "elf.R_AARCH64_P32_TLSLE_ADD_TPREL_LO12_NC": "debug/elf", - "elf.R_AARCH64_P32_TLSLE_MOVW_TPREL_G0": "debug/elf", - "elf.R_AARCH64_P32_TLSLE_MOVW_TPREL_G0_NC": "debug/elf", - "elf.R_AARCH64_P32_TLSLE_MOVW_TPREL_G1": "debug/elf", - "elf.R_AARCH64_P32_TLS_DTPMOD": "debug/elf", - "elf.R_AARCH64_P32_TLS_DTPREL": "debug/elf", - "elf.R_AARCH64_P32_TLS_TPREL": "debug/elf", - "elf.R_AARCH64_P32_TSTBR14": "debug/elf", - "elf.R_AARCH64_PREL16": "debug/elf", - "elf.R_AARCH64_PREL32": "debug/elf", - "elf.R_AARCH64_PREL64": "debug/elf", - "elf.R_AARCH64_RELATIVE": "debug/elf", - "elf.R_AARCH64_TLSDESC": "debug/elf", - "elf.R_AARCH64_TLSDESC_ADD": "debug/elf", - "elf.R_AARCH64_TLSDESC_ADD_LO12_NC": "debug/elf", - "elf.R_AARCH64_TLSDESC_ADR_PAGE21": "debug/elf", - "elf.R_AARCH64_TLSDESC_ADR_PREL21": "debug/elf", - "elf.R_AARCH64_TLSDESC_CALL": "debug/elf", - "elf.R_AARCH64_TLSDESC_LD64_LO12_NC": "debug/elf", - "elf.R_AARCH64_TLSDESC_LDR": "debug/elf", - "elf.R_AARCH64_TLSDESC_LD_PREL19": "debug/elf", - "elf.R_AARCH64_TLSDESC_OFF_G0_NC": "debug/elf", - "elf.R_AARCH64_TLSDESC_OFF_G1": "debug/elf", - "elf.R_AARCH64_TLSGD_ADD_LO12_NC": "debug/elf", - "elf.R_AARCH64_TLSGD_ADR_PAGE21": "debug/elf", - "elf.R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21": "debug/elf", - "elf.R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC": "debug/elf", - "elf.R_AARCH64_TLSIE_LD_GOTTPREL_PREL19": "debug/elf", - "elf.R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC": "debug/elf", - "elf.R_AARCH64_TLSIE_MOVW_GOTTPREL_G1": "debug/elf", - "elf.R_AARCH64_TLSLE_ADD_TPREL_HI12": "debug/elf", - "elf.R_AARCH64_TLSLE_ADD_TPREL_LO12": "debug/elf", - "elf.R_AARCH64_TLSLE_ADD_TPREL_LO12_NC": "debug/elf", - "elf.R_AARCH64_TLSLE_MOVW_TPREL_G0": "debug/elf", - "elf.R_AARCH64_TLSLE_MOVW_TPREL_G0_NC": "debug/elf", - "elf.R_AARCH64_TLSLE_MOVW_TPREL_G1": "debug/elf", - "elf.R_AARCH64_TLSLE_MOVW_TPREL_G1_NC": "debug/elf", - "elf.R_AARCH64_TLSLE_MOVW_TPREL_G2": "debug/elf", - "elf.R_AARCH64_TLS_DTPMOD64": "debug/elf", - "elf.R_AARCH64_TLS_DTPREL64": "debug/elf", - "elf.R_AARCH64_TLS_TPREL64": "debug/elf", - "elf.R_AARCH64_TSTBR14": "debug/elf", - "elf.R_ALPHA": "debug/elf", - "elf.R_ALPHA_BRADDR": "debug/elf", - "elf.R_ALPHA_COPY": "debug/elf", - "elf.R_ALPHA_GLOB_DAT": "debug/elf", - "elf.R_ALPHA_GPDISP": "debug/elf", - "elf.R_ALPHA_GPREL32": "debug/elf", - "elf.R_ALPHA_GPRELHIGH": "debug/elf", - "elf.R_ALPHA_GPRELLOW": "debug/elf", - "elf.R_ALPHA_GPVALUE": "debug/elf", - "elf.R_ALPHA_HINT": "debug/elf", - "elf.R_ALPHA_IMMED_BR_HI32": "debug/elf", - "elf.R_ALPHA_IMMED_GP_16": "debug/elf", - "elf.R_ALPHA_IMMED_GP_HI32": "debug/elf", - "elf.R_ALPHA_IMMED_LO32": "debug/elf", - "elf.R_ALPHA_IMMED_SCN_HI32": "debug/elf", - "elf.R_ALPHA_JMP_SLOT": "debug/elf", - "elf.R_ALPHA_LITERAL": "debug/elf", - "elf.R_ALPHA_LITUSE": "debug/elf", - "elf.R_ALPHA_NONE": "debug/elf", - "elf.R_ALPHA_OP_PRSHIFT": "debug/elf", - "elf.R_ALPHA_OP_PSUB": "debug/elf", - "elf.R_ALPHA_OP_PUSH": "debug/elf", - "elf.R_ALPHA_OP_STORE": "debug/elf", - "elf.R_ALPHA_REFLONG": "debug/elf", - "elf.R_ALPHA_REFQUAD": "debug/elf", - "elf.R_ALPHA_RELATIVE": "debug/elf", - "elf.R_ALPHA_SREL16": "debug/elf", - "elf.R_ALPHA_SREL32": "debug/elf", - "elf.R_ALPHA_SREL64": "debug/elf", - "elf.R_ARM": "debug/elf", - "elf.R_ARM_ABS12": "debug/elf", - "elf.R_ARM_ABS16": "debug/elf", - "elf.R_ARM_ABS32": "debug/elf", - "elf.R_ARM_ABS8": "debug/elf", - "elf.R_ARM_AMP_VCALL9": "debug/elf", - "elf.R_ARM_COPY": "debug/elf", - "elf.R_ARM_GLOB_DAT": "debug/elf", - "elf.R_ARM_GNU_VTENTRY": "debug/elf", - "elf.R_ARM_GNU_VTINHERIT": "debug/elf", - "elf.R_ARM_GOT32": "debug/elf", - "elf.R_ARM_GOTOFF": "debug/elf", - "elf.R_ARM_GOTPC": "debug/elf", - "elf.R_ARM_JUMP_SLOT": "debug/elf", - "elf.R_ARM_NONE": "debug/elf", - "elf.R_ARM_PC13": "debug/elf", - "elf.R_ARM_PC24": "debug/elf", - "elf.R_ARM_PLT32": "debug/elf", - "elf.R_ARM_RABS32": "debug/elf", - "elf.R_ARM_RBASE": "debug/elf", - "elf.R_ARM_REL32": "debug/elf", - "elf.R_ARM_RELATIVE": "debug/elf", - "elf.R_ARM_RPC24": "debug/elf", - "elf.R_ARM_RREL32": "debug/elf", - "elf.R_ARM_RSBREL32": "debug/elf", - "elf.R_ARM_SBREL32": "debug/elf", - "elf.R_ARM_SWI24": "debug/elf", - "elf.R_ARM_THM_ABS5": "debug/elf", - "elf.R_ARM_THM_PC22": "debug/elf", - "elf.R_ARM_THM_PC8": "debug/elf", - "elf.R_ARM_THM_RPC22": "debug/elf", - "elf.R_ARM_THM_SWI8": "debug/elf", - "elf.R_ARM_THM_XPC22": "debug/elf", - "elf.R_ARM_XPC25": "debug/elf", - "elf.R_INFO": "debug/elf", - "elf.R_INFO32": "debug/elf", - "elf.R_MIPS": "debug/elf", - "elf.R_MIPS_16": "debug/elf", - "elf.R_MIPS_26": "debug/elf", - "elf.R_MIPS_32": "debug/elf", - "elf.R_MIPS_64": "debug/elf", - "elf.R_MIPS_ADD_IMMEDIATE": "debug/elf", - "elf.R_MIPS_CALL16": "debug/elf", - "elf.R_MIPS_CALL_HI16": "debug/elf", - "elf.R_MIPS_CALL_LO16": "debug/elf", - "elf.R_MIPS_DELETE": "debug/elf", - "elf.R_MIPS_GOT16": "debug/elf", - "elf.R_MIPS_GOT_DISP": "debug/elf", - "elf.R_MIPS_GOT_HI16": "debug/elf", - "elf.R_MIPS_GOT_LO16": "debug/elf", - "elf.R_MIPS_GOT_OFST": "debug/elf", - "elf.R_MIPS_GOT_PAGE": "debug/elf", - "elf.R_MIPS_GPREL16": "debug/elf", - "elf.R_MIPS_GPREL32": "debug/elf", - "elf.R_MIPS_HI16": "debug/elf", - "elf.R_MIPS_HIGHER": "debug/elf", - "elf.R_MIPS_HIGHEST": "debug/elf", - "elf.R_MIPS_INSERT_A": "debug/elf", - "elf.R_MIPS_INSERT_B": "debug/elf", - "elf.R_MIPS_JALR": "debug/elf", - "elf.R_MIPS_LITERAL": "debug/elf", - "elf.R_MIPS_LO16": "debug/elf", - "elf.R_MIPS_NONE": "debug/elf", - "elf.R_MIPS_PC16": "debug/elf", - "elf.R_MIPS_PJUMP": "debug/elf", - "elf.R_MIPS_REL16": "debug/elf", - "elf.R_MIPS_REL32": "debug/elf", - "elf.R_MIPS_RELGOT": "debug/elf", - "elf.R_MIPS_SCN_DISP": "debug/elf", - "elf.R_MIPS_SHIFT5": "debug/elf", - "elf.R_MIPS_SHIFT6": "debug/elf", - "elf.R_MIPS_SUB": "debug/elf", - "elf.R_MIPS_TLS_DTPMOD32": "debug/elf", - "elf.R_MIPS_TLS_DTPMOD64": "debug/elf", - "elf.R_MIPS_TLS_DTPREL32": "debug/elf", - "elf.R_MIPS_TLS_DTPREL64": "debug/elf", - "elf.R_MIPS_TLS_DTPREL_HI16": "debug/elf", - "elf.R_MIPS_TLS_DTPREL_LO16": "debug/elf", - "elf.R_MIPS_TLS_GD": "debug/elf", - "elf.R_MIPS_TLS_GOTTPREL": "debug/elf", - "elf.R_MIPS_TLS_LDM": "debug/elf", - "elf.R_MIPS_TLS_TPREL32": "debug/elf", - "elf.R_MIPS_TLS_TPREL64": "debug/elf", - "elf.R_MIPS_TLS_TPREL_HI16": "debug/elf", - "elf.R_MIPS_TLS_TPREL_LO16": "debug/elf", - "elf.R_PPC": "debug/elf", - "elf.R_PPC64": "debug/elf", - "elf.R_PPC64_ADDR14": "debug/elf", - "elf.R_PPC64_ADDR14_BRNTAKEN": "debug/elf", - "elf.R_PPC64_ADDR14_BRTAKEN": "debug/elf", - "elf.R_PPC64_ADDR16": "debug/elf", - "elf.R_PPC64_ADDR16_DS": "debug/elf", - "elf.R_PPC64_ADDR16_HA": "debug/elf", - "elf.R_PPC64_ADDR16_HI": "debug/elf", - "elf.R_PPC64_ADDR16_HIGHER": "debug/elf", - "elf.R_PPC64_ADDR16_HIGHERA": "debug/elf", - "elf.R_PPC64_ADDR16_HIGHEST": "debug/elf", - "elf.R_PPC64_ADDR16_HIGHESTA": "debug/elf", - "elf.R_PPC64_ADDR16_LO": "debug/elf", - "elf.R_PPC64_ADDR16_LO_DS": "debug/elf", - "elf.R_PPC64_ADDR24": "debug/elf", - "elf.R_PPC64_ADDR32": "debug/elf", - "elf.R_PPC64_ADDR64": "debug/elf", - "elf.R_PPC64_DTPMOD64": "debug/elf", - "elf.R_PPC64_DTPREL16": "debug/elf", - "elf.R_PPC64_DTPREL16_DS": "debug/elf", - "elf.R_PPC64_DTPREL16_HA": "debug/elf", - "elf.R_PPC64_DTPREL16_HI": "debug/elf", - "elf.R_PPC64_DTPREL16_HIGHER": "debug/elf", - "elf.R_PPC64_DTPREL16_HIGHERA": "debug/elf", - "elf.R_PPC64_DTPREL16_HIGHEST": "debug/elf", - "elf.R_PPC64_DTPREL16_HIGHESTA": "debug/elf", - "elf.R_PPC64_DTPREL16_LO": "debug/elf", - "elf.R_PPC64_DTPREL16_LO_DS": "debug/elf", - "elf.R_PPC64_DTPREL64": "debug/elf", - "elf.R_PPC64_GOT16": "debug/elf", - "elf.R_PPC64_GOT16_DS": "debug/elf", - "elf.R_PPC64_GOT16_HA": "debug/elf", - "elf.R_PPC64_GOT16_HI": "debug/elf", - "elf.R_PPC64_GOT16_LO": "debug/elf", - "elf.R_PPC64_GOT16_LO_DS": "debug/elf", - "elf.R_PPC64_GOT_DTPREL16_DS": "debug/elf", - "elf.R_PPC64_GOT_DTPREL16_HA": "debug/elf", - "elf.R_PPC64_GOT_DTPREL16_HI": "debug/elf", - "elf.R_PPC64_GOT_DTPREL16_LO_DS": "debug/elf", - "elf.R_PPC64_GOT_TLSGD16": "debug/elf", - "elf.R_PPC64_GOT_TLSGD16_HA": "debug/elf", - "elf.R_PPC64_GOT_TLSGD16_HI": "debug/elf", - "elf.R_PPC64_GOT_TLSGD16_LO": "debug/elf", - "elf.R_PPC64_GOT_TLSLD16": "debug/elf", - "elf.R_PPC64_GOT_TLSLD16_HA": "debug/elf", - "elf.R_PPC64_GOT_TLSLD16_HI": "debug/elf", - "elf.R_PPC64_GOT_TLSLD16_LO": "debug/elf", - "elf.R_PPC64_GOT_TPREL16_DS": "debug/elf", - "elf.R_PPC64_GOT_TPREL16_HA": "debug/elf", - "elf.R_PPC64_GOT_TPREL16_HI": "debug/elf", - "elf.R_PPC64_GOT_TPREL16_LO_DS": "debug/elf", - "elf.R_PPC64_JMP_SLOT": "debug/elf", - "elf.R_PPC64_NONE": "debug/elf", - "elf.R_PPC64_REL14": "debug/elf", - "elf.R_PPC64_REL14_BRNTAKEN": "debug/elf", - "elf.R_PPC64_REL14_BRTAKEN": "debug/elf", - "elf.R_PPC64_REL16": "debug/elf", - "elf.R_PPC64_REL16_HA": "debug/elf", - "elf.R_PPC64_REL16_HI": "debug/elf", - "elf.R_PPC64_REL16_LO": "debug/elf", - "elf.R_PPC64_REL24": "debug/elf", - "elf.R_PPC64_REL32": "debug/elf", - "elf.R_PPC64_REL64": "debug/elf", - "elf.R_PPC64_TLS": "debug/elf", - "elf.R_PPC64_TLSGD": "debug/elf", - "elf.R_PPC64_TLSLD": "debug/elf", - "elf.R_PPC64_TOC": "debug/elf", - "elf.R_PPC64_TOC16": "debug/elf", - "elf.R_PPC64_TOC16_DS": "debug/elf", - "elf.R_PPC64_TOC16_HA": "debug/elf", - "elf.R_PPC64_TOC16_HI": "debug/elf", - "elf.R_PPC64_TOC16_LO": "debug/elf", - "elf.R_PPC64_TOC16_LO_DS": "debug/elf", - "elf.R_PPC64_TPREL16": "debug/elf", - "elf.R_PPC64_TPREL16_DS": "debug/elf", - "elf.R_PPC64_TPREL16_HA": "debug/elf", - "elf.R_PPC64_TPREL16_HI": "debug/elf", - "elf.R_PPC64_TPREL16_HIGHER": "debug/elf", - "elf.R_PPC64_TPREL16_HIGHERA": "debug/elf", - "elf.R_PPC64_TPREL16_HIGHEST": "debug/elf", - "elf.R_PPC64_TPREL16_HIGHESTA": "debug/elf", - "elf.R_PPC64_TPREL16_LO": "debug/elf", - "elf.R_PPC64_TPREL16_LO_DS": "debug/elf", - "elf.R_PPC64_TPREL64": "debug/elf", - "elf.R_PPC_ADDR14": "debug/elf", - "elf.R_PPC_ADDR14_BRNTAKEN": "debug/elf", - "elf.R_PPC_ADDR14_BRTAKEN": "debug/elf", - "elf.R_PPC_ADDR16": "debug/elf", - "elf.R_PPC_ADDR16_HA": "debug/elf", - "elf.R_PPC_ADDR16_HI": "debug/elf", - "elf.R_PPC_ADDR16_LO": "debug/elf", - "elf.R_PPC_ADDR24": "debug/elf", - "elf.R_PPC_ADDR32": "debug/elf", - "elf.R_PPC_COPY": "debug/elf", - "elf.R_PPC_DTPMOD32": "debug/elf", - "elf.R_PPC_DTPREL16": "debug/elf", - "elf.R_PPC_DTPREL16_HA": "debug/elf", - "elf.R_PPC_DTPREL16_HI": "debug/elf", - "elf.R_PPC_DTPREL16_LO": "debug/elf", - "elf.R_PPC_DTPREL32": "debug/elf", - "elf.R_PPC_EMB_BIT_FLD": "debug/elf", - "elf.R_PPC_EMB_MRKREF": "debug/elf", - "elf.R_PPC_EMB_NADDR16": "debug/elf", - "elf.R_PPC_EMB_NADDR16_HA": "debug/elf", - "elf.R_PPC_EMB_NADDR16_HI": "debug/elf", - "elf.R_PPC_EMB_NADDR16_LO": "debug/elf", - "elf.R_PPC_EMB_NADDR32": "debug/elf", - "elf.R_PPC_EMB_RELSDA": "debug/elf", - "elf.R_PPC_EMB_RELSEC16": "debug/elf", - "elf.R_PPC_EMB_RELST_HA": "debug/elf", - "elf.R_PPC_EMB_RELST_HI": "debug/elf", - "elf.R_PPC_EMB_RELST_LO": "debug/elf", - "elf.R_PPC_EMB_SDA21": "debug/elf", - "elf.R_PPC_EMB_SDA2I16": "debug/elf", - "elf.R_PPC_EMB_SDA2REL": "debug/elf", - "elf.R_PPC_EMB_SDAI16": "debug/elf", - "elf.R_PPC_GLOB_DAT": "debug/elf", - "elf.R_PPC_GOT16": "debug/elf", - "elf.R_PPC_GOT16_HA": "debug/elf", - "elf.R_PPC_GOT16_HI": "debug/elf", - "elf.R_PPC_GOT16_LO": "debug/elf", - "elf.R_PPC_GOT_TLSGD16": "debug/elf", - "elf.R_PPC_GOT_TLSGD16_HA": "debug/elf", - "elf.R_PPC_GOT_TLSGD16_HI": "debug/elf", - "elf.R_PPC_GOT_TLSGD16_LO": "debug/elf", - "elf.R_PPC_GOT_TLSLD16": "debug/elf", - "elf.R_PPC_GOT_TLSLD16_HA": "debug/elf", - "elf.R_PPC_GOT_TLSLD16_HI": "debug/elf", - "elf.R_PPC_GOT_TLSLD16_LO": "debug/elf", - "elf.R_PPC_GOT_TPREL16": "debug/elf", - "elf.R_PPC_GOT_TPREL16_HA": "debug/elf", - "elf.R_PPC_GOT_TPREL16_HI": "debug/elf", - "elf.R_PPC_GOT_TPREL16_LO": "debug/elf", - "elf.R_PPC_JMP_SLOT": "debug/elf", - "elf.R_PPC_LOCAL24PC": "debug/elf", - "elf.R_PPC_NONE": "debug/elf", - "elf.R_PPC_PLT16_HA": "debug/elf", - "elf.R_PPC_PLT16_HI": "debug/elf", - "elf.R_PPC_PLT16_LO": "debug/elf", - "elf.R_PPC_PLT32": "debug/elf", - "elf.R_PPC_PLTREL24": "debug/elf", - "elf.R_PPC_PLTREL32": "debug/elf", - "elf.R_PPC_REL14": "debug/elf", - "elf.R_PPC_REL14_BRNTAKEN": "debug/elf", - "elf.R_PPC_REL14_BRTAKEN": "debug/elf", - "elf.R_PPC_REL24": "debug/elf", - "elf.R_PPC_REL32": "debug/elf", - "elf.R_PPC_RELATIVE": "debug/elf", - "elf.R_PPC_SDAREL16": "debug/elf", - "elf.R_PPC_SECTOFF": "debug/elf", - "elf.R_PPC_SECTOFF_HA": "debug/elf", - "elf.R_PPC_SECTOFF_HI": "debug/elf", - "elf.R_PPC_SECTOFF_LO": "debug/elf", - "elf.R_PPC_TLS": "debug/elf", - "elf.R_PPC_TPREL16": "debug/elf", - "elf.R_PPC_TPREL16_HA": "debug/elf", - "elf.R_PPC_TPREL16_HI": "debug/elf", - "elf.R_PPC_TPREL16_LO": "debug/elf", - "elf.R_PPC_TPREL32": "debug/elf", - "elf.R_PPC_UADDR16": "debug/elf", - "elf.R_PPC_UADDR32": "debug/elf", - "elf.R_SPARC": "debug/elf", - "elf.R_SPARC_10": "debug/elf", - "elf.R_SPARC_11": "debug/elf", - "elf.R_SPARC_13": "debug/elf", - "elf.R_SPARC_16": "debug/elf", - "elf.R_SPARC_22": "debug/elf", - "elf.R_SPARC_32": "debug/elf", - "elf.R_SPARC_5": "debug/elf", - "elf.R_SPARC_6": "debug/elf", - "elf.R_SPARC_64": "debug/elf", - "elf.R_SPARC_7": "debug/elf", - "elf.R_SPARC_8": "debug/elf", - "elf.R_SPARC_COPY": "debug/elf", - "elf.R_SPARC_DISP16": "debug/elf", - "elf.R_SPARC_DISP32": "debug/elf", - "elf.R_SPARC_DISP64": "debug/elf", - "elf.R_SPARC_DISP8": "debug/elf", - "elf.R_SPARC_GLOB_DAT": "debug/elf", - "elf.R_SPARC_GLOB_JMP": "debug/elf", - "elf.R_SPARC_GOT10": "debug/elf", - "elf.R_SPARC_GOT13": "debug/elf", - "elf.R_SPARC_GOT22": "debug/elf", - "elf.R_SPARC_H44": "debug/elf", - "elf.R_SPARC_HH22": "debug/elf", - "elf.R_SPARC_HI22": "debug/elf", - "elf.R_SPARC_HIPLT22": "debug/elf", - "elf.R_SPARC_HIX22": "debug/elf", - "elf.R_SPARC_HM10": "debug/elf", - "elf.R_SPARC_JMP_SLOT": "debug/elf", - "elf.R_SPARC_L44": "debug/elf", - "elf.R_SPARC_LM22": "debug/elf", - "elf.R_SPARC_LO10": "debug/elf", - "elf.R_SPARC_LOPLT10": "debug/elf", - "elf.R_SPARC_LOX10": "debug/elf", - "elf.R_SPARC_M44": "debug/elf", - "elf.R_SPARC_NONE": "debug/elf", - "elf.R_SPARC_OLO10": "debug/elf", - "elf.R_SPARC_PC10": "debug/elf", - "elf.R_SPARC_PC22": "debug/elf", - "elf.R_SPARC_PCPLT10": "debug/elf", - "elf.R_SPARC_PCPLT22": "debug/elf", - "elf.R_SPARC_PCPLT32": "debug/elf", - "elf.R_SPARC_PC_HH22": "debug/elf", - "elf.R_SPARC_PC_HM10": "debug/elf", - "elf.R_SPARC_PC_LM22": "debug/elf", - "elf.R_SPARC_PLT32": "debug/elf", - "elf.R_SPARC_PLT64": "debug/elf", - "elf.R_SPARC_REGISTER": "debug/elf", - "elf.R_SPARC_RELATIVE": "debug/elf", - "elf.R_SPARC_UA16": "debug/elf", - "elf.R_SPARC_UA32": "debug/elf", - "elf.R_SPARC_UA64": "debug/elf", - "elf.R_SPARC_WDISP16": "debug/elf", - "elf.R_SPARC_WDISP19": "debug/elf", - "elf.R_SPARC_WDISP22": "debug/elf", - "elf.R_SPARC_WDISP30": "debug/elf", - "elf.R_SPARC_WPLT30": "debug/elf", - "elf.R_SYM32": "debug/elf", - "elf.R_SYM64": "debug/elf", - "elf.R_TYPE32": "debug/elf", - "elf.R_TYPE64": "debug/elf", - "elf.R_X86_64": "debug/elf", - "elf.R_X86_64_16": "debug/elf", - "elf.R_X86_64_32": "debug/elf", - "elf.R_X86_64_32S": "debug/elf", - "elf.R_X86_64_64": "debug/elf", - "elf.R_X86_64_8": "debug/elf", - "elf.R_X86_64_COPY": "debug/elf", - "elf.R_X86_64_DTPMOD64": "debug/elf", - "elf.R_X86_64_DTPOFF32": "debug/elf", - "elf.R_X86_64_DTPOFF64": "debug/elf", - "elf.R_X86_64_GLOB_DAT": "debug/elf", - "elf.R_X86_64_GOT32": "debug/elf", - "elf.R_X86_64_GOTPCREL": "debug/elf", - "elf.R_X86_64_GOTTPOFF": "debug/elf", - "elf.R_X86_64_JMP_SLOT": "debug/elf", - "elf.R_X86_64_NONE": "debug/elf", - "elf.R_X86_64_PC16": "debug/elf", - "elf.R_X86_64_PC32": "debug/elf", - "elf.R_X86_64_PC8": "debug/elf", - "elf.R_X86_64_PLT32": "debug/elf", - "elf.R_X86_64_RELATIVE": "debug/elf", - "elf.R_X86_64_TLSGD": "debug/elf", - "elf.R_X86_64_TLSLD": "debug/elf", - "elf.R_X86_64_TPOFF32": "debug/elf", - "elf.R_X86_64_TPOFF64": "debug/elf", - "elf.Rel32": "debug/elf", - "elf.Rel64": "debug/elf", - "elf.Rela32": "debug/elf", - "elf.Rela64": "debug/elf", - "elf.SHF_ALLOC": "debug/elf", - "elf.SHF_COMPRESSED": "debug/elf", - "elf.SHF_EXECINSTR": "debug/elf", - "elf.SHF_GROUP": "debug/elf", - "elf.SHF_INFO_LINK": "debug/elf", - "elf.SHF_LINK_ORDER": "debug/elf", - "elf.SHF_MASKOS": "debug/elf", - "elf.SHF_MASKPROC": "debug/elf", - "elf.SHF_MERGE": "debug/elf", - "elf.SHF_OS_NONCONFORMING": "debug/elf", - "elf.SHF_STRINGS": "debug/elf", - "elf.SHF_TLS": "debug/elf", - "elf.SHF_WRITE": "debug/elf", - "elf.SHN_ABS": "debug/elf", - "elf.SHN_COMMON": "debug/elf", - "elf.SHN_HIOS": "debug/elf", - "elf.SHN_HIPROC": "debug/elf", - "elf.SHN_HIRESERVE": "debug/elf", - "elf.SHN_LOOS": "debug/elf", - "elf.SHN_LOPROC": "debug/elf", - "elf.SHN_LORESERVE": "debug/elf", - "elf.SHN_UNDEF": "debug/elf", - "elf.SHN_XINDEX": "debug/elf", - "elf.SHT_DYNAMIC": "debug/elf", - "elf.SHT_DYNSYM": "debug/elf", - "elf.SHT_FINI_ARRAY": "debug/elf", - "elf.SHT_GNU_ATTRIBUTES": "debug/elf", - "elf.SHT_GNU_HASH": "debug/elf", - "elf.SHT_GNU_LIBLIST": "debug/elf", - "elf.SHT_GNU_VERDEF": "debug/elf", - "elf.SHT_GNU_VERNEED": "debug/elf", - "elf.SHT_GNU_VERSYM": "debug/elf", - "elf.SHT_GROUP": "debug/elf", - "elf.SHT_HASH": "debug/elf", - "elf.SHT_HIOS": "debug/elf", - "elf.SHT_HIPROC": "debug/elf", - "elf.SHT_HIUSER": "debug/elf", - "elf.SHT_INIT_ARRAY": "debug/elf", - "elf.SHT_LOOS": "debug/elf", - "elf.SHT_LOPROC": "debug/elf", - "elf.SHT_LOUSER": "debug/elf", - "elf.SHT_NOBITS": "debug/elf", - "elf.SHT_NOTE": "debug/elf", - "elf.SHT_NULL": "debug/elf", - "elf.SHT_PREINIT_ARRAY": "debug/elf", - "elf.SHT_PROGBITS": "debug/elf", - "elf.SHT_REL": "debug/elf", - "elf.SHT_RELA": "debug/elf", - "elf.SHT_SHLIB": "debug/elf", - "elf.SHT_STRTAB": "debug/elf", - "elf.SHT_SYMTAB": "debug/elf", - "elf.SHT_SYMTAB_SHNDX": "debug/elf", - "elf.STB_GLOBAL": "debug/elf", - "elf.STB_HIOS": "debug/elf", - "elf.STB_HIPROC": "debug/elf", - "elf.STB_LOCAL": "debug/elf", - "elf.STB_LOOS": "debug/elf", - "elf.STB_LOPROC": "debug/elf", - "elf.STB_WEAK": "debug/elf", - "elf.STT_COMMON": "debug/elf", - "elf.STT_FILE": "debug/elf", - "elf.STT_FUNC": "debug/elf", - "elf.STT_HIOS": "debug/elf", - "elf.STT_HIPROC": "debug/elf", - "elf.STT_LOOS": "debug/elf", - "elf.STT_LOPROC": "debug/elf", - "elf.STT_NOTYPE": "debug/elf", - "elf.STT_OBJECT": "debug/elf", - "elf.STT_SECTION": "debug/elf", - "elf.STT_TLS": "debug/elf", - "elf.STV_DEFAULT": "debug/elf", - "elf.STV_HIDDEN": "debug/elf", - "elf.STV_INTERNAL": "debug/elf", - "elf.STV_PROTECTED": "debug/elf", - "elf.ST_BIND": "debug/elf", - "elf.ST_INFO": "debug/elf", - "elf.ST_TYPE": "debug/elf", - "elf.ST_VISIBILITY": "debug/elf", - "elf.Section": "debug/elf", - "elf.Section32": "debug/elf", - "elf.Section64": "debug/elf", - "elf.SectionFlag": "debug/elf", - "elf.SectionHeader": "debug/elf", - "elf.SectionIndex": "debug/elf", - "elf.SectionType": "debug/elf", - "elf.Sym32": "debug/elf", - "elf.Sym32Size": "debug/elf", - "elf.Sym64": "debug/elf", - "elf.Sym64Size": "debug/elf", - "elf.SymBind": "debug/elf", - "elf.SymType": "debug/elf", - "elf.SymVis": "debug/elf", - "elf.Symbol": "debug/elf", - "elf.Type": "debug/elf", - "elf.Version": "debug/elf", - "elliptic.Curve": "crypto/elliptic", - "elliptic.CurveParams": "crypto/elliptic", - "elliptic.GenerateKey": "crypto/elliptic", - "elliptic.Marshal": "crypto/elliptic", - "elliptic.P224": "crypto/elliptic", - "elliptic.P256": "crypto/elliptic", - "elliptic.P384": "crypto/elliptic", - "elliptic.P521": "crypto/elliptic", - "elliptic.Unmarshal": "crypto/elliptic", - "encoding.BinaryMarshaler": "encoding", - "encoding.BinaryUnmarshaler": "encoding", - "encoding.TextMarshaler": "encoding", - "encoding.TextUnmarshaler": "encoding", - "errors.New": "errors", - "exec.Cmd": "os/exec", - "exec.Command": "os/exec", - "exec.CommandContext": "os/exec", - "exec.ErrNotFound": "os/exec", - "exec.Error": "os/exec", - "exec.ExitError": "os/exec", - "exec.LookPath": "os/exec", - "expvar.Do": "expvar", - "expvar.Float": "expvar", - "expvar.Func": "expvar", - "expvar.Get": "expvar", - "expvar.Int": "expvar", - "expvar.KeyValue": "expvar", - "expvar.Map": "expvar", - "expvar.NewFloat": "expvar", - "expvar.NewInt": "expvar", - "expvar.NewMap": "expvar", - "expvar.NewString": "expvar", - "expvar.Publish": "expvar", - "expvar.String": "expvar", - "expvar.Var": "expvar", - "fcgi.ErrConnClosed": "net/http/fcgi", - "fcgi.ErrRequestAborted": "net/http/fcgi", - "fcgi.Serve": "net/http/fcgi", - "filepath.Abs": "path/filepath", - "filepath.Base": "path/filepath", - "filepath.Clean": "path/filepath", - "filepath.Dir": "path/filepath", - "filepath.ErrBadPattern": "path/filepath", - "filepath.EvalSymlinks": "path/filepath", - "filepath.Ext": "path/filepath", - "filepath.FromSlash": "path/filepath", - "filepath.Glob": "path/filepath", - "filepath.HasPrefix": "path/filepath", - "filepath.IsAbs": "path/filepath", - "filepath.Join": "path/filepath", - "filepath.ListSeparator": "path/filepath", - "filepath.Match": "path/filepath", - "filepath.Rel": "path/filepath", - "filepath.Separator": "path/filepath", - "filepath.SkipDir": "path/filepath", - "filepath.Split": "path/filepath", - "filepath.SplitList": "path/filepath", - "filepath.ToSlash": "path/filepath", - "filepath.VolumeName": "path/filepath", - "filepath.Walk": "path/filepath", - "filepath.WalkFunc": "path/filepath", - "flag.Arg": "flag", - "flag.Args": "flag", - "flag.Bool": "flag", - "flag.BoolVar": "flag", - "flag.CommandLine": "flag", - "flag.ContinueOnError": "flag", - "flag.Duration": "flag", - "flag.DurationVar": "flag", - "flag.ErrHelp": "flag", - "flag.ErrorHandling": "flag", - "flag.ExitOnError": "flag", - "flag.Flag": "flag", - "flag.FlagSet": "flag", - "flag.Float64": "flag", - "flag.Float64Var": "flag", - "flag.Getter": "flag", - "flag.Int": "flag", - "flag.Int64": "flag", - "flag.Int64Var": "flag", - "flag.IntVar": "flag", - "flag.Lookup": "flag", - "flag.NArg": "flag", - "flag.NFlag": "flag", - "flag.NewFlagSet": "flag", - "flag.PanicOnError": "flag", - "flag.Parse": "flag", - "flag.Parsed": "flag", - "flag.PrintDefaults": "flag", - "flag.Set": "flag", - "flag.String": "flag", - "flag.StringVar": "flag", - "flag.Uint": "flag", - "flag.Uint64": "flag", - "flag.Uint64Var": "flag", - "flag.UintVar": "flag", - "flag.UnquoteUsage": "flag", - "flag.Usage": "flag", - "flag.Value": "flag", - "flag.Var": "flag", - "flag.Visit": "flag", - "flag.VisitAll": "flag", - "flate.BestCompression": "compress/flate", - "flate.BestSpeed": "compress/flate", - "flate.CorruptInputError": "compress/flate", - "flate.DefaultCompression": "compress/flate", - "flate.HuffmanOnly": "compress/flate", - "flate.InternalError": "compress/flate", - "flate.NewReader": "compress/flate", - "flate.NewReaderDict": "compress/flate", - "flate.NewWriter": "compress/flate", - "flate.NewWriterDict": "compress/flate", - "flate.NoCompression": "compress/flate", - "flate.ReadError": "compress/flate", - "flate.Reader": "compress/flate", - "flate.Resetter": "compress/flate", - "flate.WriteError": "compress/flate", - "flate.Writer": "compress/flate", - "fmt.Errorf": "fmt", - "fmt.Formatter": "fmt", - "fmt.Fprint": "fmt", - "fmt.Fprintf": "fmt", - "fmt.Fprintln": "fmt", - "fmt.Fscan": "fmt", - "fmt.Fscanf": "fmt", - "fmt.Fscanln": "fmt", - "fmt.GoStringer": "fmt", - "fmt.Print": "fmt", - "fmt.Printf": "fmt", - "fmt.Println": "fmt", - "fmt.Scan": "fmt", - "fmt.ScanState": "fmt", - "fmt.Scanf": "fmt", - "fmt.Scanln": "fmt", - "fmt.Scanner": "fmt", - "fmt.Sprint": "fmt", - "fmt.Sprintf": "fmt", - "fmt.Sprintln": "fmt", - "fmt.Sscan": "fmt", - "fmt.Sscanf": "fmt", - "fmt.Sscanln": "fmt", - "fmt.State": "fmt", - "fmt.Stringer": "fmt", - "fnv.New32": "hash/fnv", - "fnv.New32a": "hash/fnv", - "fnv.New64": "hash/fnv", - "fnv.New64a": "hash/fnv", - "format.Node": "go/format", - "format.Source": "go/format", - "gif.Decode": "image/gif", - "gif.DecodeAll": "image/gif", - "gif.DecodeConfig": "image/gif", - "gif.DisposalBackground": "image/gif", - "gif.DisposalNone": "image/gif", - "gif.DisposalPrevious": "image/gif", - "gif.Encode": "image/gif", - "gif.EncodeAll": "image/gif", - "gif.GIF": "image/gif", - "gif.Options": "image/gif", - "gob.CommonType": "encoding/gob", - "gob.Decoder": "encoding/gob", - "gob.Encoder": "encoding/gob", - "gob.GobDecoder": "encoding/gob", - "gob.GobEncoder": "encoding/gob", - "gob.NewDecoder": "encoding/gob", - "gob.NewEncoder": "encoding/gob", - "gob.Register": "encoding/gob", - "gob.RegisterName": "encoding/gob", - "gosym.DecodingError": "debug/gosym", - "gosym.Func": "debug/gosym", - "gosym.LineTable": "debug/gosym", - "gosym.NewLineTable": "debug/gosym", - "gosym.NewTable": "debug/gosym", - "gosym.Obj": "debug/gosym", - "gosym.Sym": "debug/gosym", - "gosym.Table": "debug/gosym", - "gosym.UnknownFileError": "debug/gosym", - "gosym.UnknownLineError": "debug/gosym", - "gzip.BestCompression": "compress/gzip", - "gzip.BestSpeed": "compress/gzip", - "gzip.DefaultCompression": "compress/gzip", - "gzip.ErrChecksum": "compress/gzip", - "gzip.ErrHeader": "compress/gzip", - "gzip.Header": "compress/gzip", - "gzip.NewReader": "compress/gzip", - "gzip.NewWriter": "compress/gzip", - "gzip.NewWriterLevel": "compress/gzip", - "gzip.NoCompression": "compress/gzip", - "gzip.Reader": "compress/gzip", - "gzip.Writer": "compress/gzip", - "hash.Hash": "hash", - "hash.Hash32": "hash", - "hash.Hash64": "hash", - "heap.Fix": "container/heap", - "heap.Init": "container/heap", - "heap.Interface": "container/heap", - "heap.Pop": "container/heap", - "heap.Push": "container/heap", - "heap.Remove": "container/heap", - "hex.Decode": "encoding/hex", - "hex.DecodeString": "encoding/hex", - "hex.DecodedLen": "encoding/hex", - "hex.Dump": "encoding/hex", - "hex.Dumper": "encoding/hex", - "hex.Encode": "encoding/hex", - "hex.EncodeToString": "encoding/hex", - "hex.EncodedLen": "encoding/hex", - "hex.ErrLength": "encoding/hex", - "hex.InvalidByteError": "encoding/hex", - "hmac.Equal": "crypto/hmac", - "hmac.New": "crypto/hmac", - "html.EscapeString": "html", - "html.UnescapeString": "html", - "http.CanonicalHeaderKey": "net/http", - "http.Client": "net/http", - "http.CloseNotifier": "net/http", - "http.ConnState": "net/http", - "http.Cookie": "net/http", - "http.CookieJar": "net/http", - "http.DefaultClient": "net/http", - "http.DefaultMaxHeaderBytes": "net/http", - "http.DefaultMaxIdleConnsPerHost": "net/http", - "http.DefaultServeMux": "net/http", - "http.DefaultTransport": "net/http", - "http.DetectContentType": "net/http", - "http.Dir": "net/http", - "http.ErrBodyNotAllowed": "net/http", - "http.ErrBodyReadAfterClose": "net/http", - "http.ErrContentLength": "net/http", - "http.ErrHandlerTimeout": "net/http", - "http.ErrHeaderTooLong": "net/http", - "http.ErrHijacked": "net/http", - "http.ErrLineTooLong": "net/http", - "http.ErrMissingBoundary": "net/http", - "http.ErrMissingContentLength": "net/http", - "http.ErrMissingFile": "net/http", - "http.ErrNoCookie": "net/http", - "http.ErrNoLocation": "net/http", - "http.ErrNotMultipart": "net/http", - "http.ErrNotSupported": "net/http", - "http.ErrShortBody": "net/http", - "http.ErrSkipAltProtocol": "net/http", - "http.ErrUnexpectedTrailer": "net/http", - "http.ErrUseLastResponse": "net/http", - "http.ErrWriteAfterFlush": "net/http", - "http.Error": "net/http", - "http.File": "net/http", - "http.FileServer": "net/http", - "http.FileSystem": "net/http", - "http.Flusher": "net/http", - "http.Get": "net/http", - "http.Handle": "net/http", - "http.HandleFunc": "net/http", - "http.Handler": "net/http", - "http.HandlerFunc": "net/http", - "http.Head": "net/http", - "http.Header": "net/http", - "http.Hijacker": "net/http", - "http.ListenAndServe": "net/http", - "http.ListenAndServeTLS": "net/http", - "http.LocalAddrContextKey": "net/http", - "http.MaxBytesReader": "net/http", - "http.MethodConnect": "net/http", - "http.MethodDelete": "net/http", - "http.MethodGet": "net/http", - "http.MethodHead": "net/http", - "http.MethodOptions": "net/http", - "http.MethodPatch": "net/http", - "http.MethodPost": "net/http", - "http.MethodPut": "net/http", - "http.MethodTrace": "net/http", - "http.NewFileTransport": "net/http", - "http.NewRequest": "net/http", - "http.NewServeMux": "net/http", - "http.NotFound": "net/http", - "http.NotFoundHandler": "net/http", - "http.ParseHTTPVersion": "net/http", - "http.ParseTime": "net/http", - "http.Post": "net/http", - "http.PostForm": "net/http", - "http.ProtocolError": "net/http", - "http.ProxyFromEnvironment": "net/http", - "http.ProxyURL": "net/http", - "http.ReadRequest": "net/http", - "http.ReadResponse": "net/http", - "http.Redirect": "net/http", - "http.RedirectHandler": "net/http", - "http.Request": "net/http", - "http.Response": "net/http", - "http.ResponseWriter": "net/http", - "http.RoundTripper": "net/http", - "http.Serve": "net/http", - "http.ServeContent": "net/http", - "http.ServeFile": "net/http", - "http.ServeMux": "net/http", - "http.Server": "net/http", - "http.ServerContextKey": "net/http", - "http.SetCookie": "net/http", - "http.StateActive": "net/http", - "http.StateClosed": "net/http", - "http.StateHijacked": "net/http", - "http.StateIdle": "net/http", - "http.StateNew": "net/http", - "http.StatusAccepted": "net/http", - "http.StatusAlreadyReported": "net/http", - "http.StatusBadGateway": "net/http", - "http.StatusBadRequest": "net/http", - "http.StatusConflict": "net/http", - "http.StatusContinue": "net/http", - "http.StatusCreated": "net/http", - "http.StatusExpectationFailed": "net/http", - "http.StatusFailedDependency": "net/http", - "http.StatusForbidden": "net/http", - "http.StatusFound": "net/http", - "http.StatusGatewayTimeout": "net/http", - "http.StatusGone": "net/http", - "http.StatusHTTPVersionNotSupported": "net/http", - "http.StatusIMUsed": "net/http", - "http.StatusInsufficientStorage": "net/http", - "http.StatusInternalServerError": "net/http", - "http.StatusLengthRequired": "net/http", - "http.StatusLocked": "net/http", - "http.StatusLoopDetected": "net/http", - "http.StatusMethodNotAllowed": "net/http", - "http.StatusMovedPermanently": "net/http", - "http.StatusMultiStatus": "net/http", - "http.StatusMultipleChoices": "net/http", - "http.StatusNetworkAuthenticationRequired": "net/http", - "http.StatusNoContent": "net/http", - "http.StatusNonAuthoritativeInfo": "net/http", - "http.StatusNotAcceptable": "net/http", - "http.StatusNotExtended": "net/http", - "http.StatusNotFound": "net/http", - "http.StatusNotImplemented": "net/http", - "http.StatusNotModified": "net/http", - "http.StatusOK": "net/http", - "http.StatusPartialContent": "net/http", - "http.StatusPaymentRequired": "net/http", - "http.StatusPermanentRedirect": "net/http", - "http.StatusPreconditionFailed": "net/http", - "http.StatusPreconditionRequired": "net/http", - "http.StatusProcessing": "net/http", - "http.StatusProxyAuthRequired": "net/http", - "http.StatusRequestEntityTooLarge": "net/http", - "http.StatusRequestHeaderFieldsTooLarge": "net/http", - "http.StatusRequestTimeout": "net/http", - "http.StatusRequestURITooLong": "net/http", - "http.StatusRequestedRangeNotSatisfiable": "net/http", - "http.StatusResetContent": "net/http", - "http.StatusSeeOther": "net/http", - "http.StatusServiceUnavailable": "net/http", - "http.StatusSwitchingProtocols": "net/http", - "http.StatusTeapot": "net/http", - "http.StatusTemporaryRedirect": "net/http", - "http.StatusText": "net/http", - "http.StatusTooManyRequests": "net/http", - "http.StatusUnauthorized": "net/http", - "http.StatusUnavailableForLegalReasons": "net/http", - "http.StatusUnprocessableEntity": "net/http", - "http.StatusUnsupportedMediaType": "net/http", - "http.StatusUpgradeRequired": "net/http", - "http.StatusUseProxy": "net/http", - "http.StatusVariantAlsoNegotiates": "net/http", - "http.StripPrefix": "net/http", - "http.TimeFormat": "net/http", - "http.TimeoutHandler": "net/http", - "http.Transport": "net/http", - "httptest.DefaultRemoteAddr": "net/http/httptest", - "httptest.NewRecorder": "net/http/httptest", - "httptest.NewRequest": "net/http/httptest", - "httptest.NewServer": "net/http/httptest", - "httptest.NewTLSServer": "net/http/httptest", - "httptest.NewUnstartedServer": "net/http/httptest", - "httptest.ResponseRecorder": "net/http/httptest", - "httptest.Server": "net/http/httptest", - "httptrace.ClientTrace": "net/http/httptrace", - "httptrace.ContextClientTrace": "net/http/httptrace", - "httptrace.DNSDoneInfo": "net/http/httptrace", - "httptrace.DNSStartInfo": "net/http/httptrace", - "httptrace.GotConnInfo": "net/http/httptrace", - "httptrace.WithClientTrace": "net/http/httptrace", - "httptrace.WroteRequestInfo": "net/http/httptrace", - "httputil.BufferPool": "net/http/httputil", - "httputil.ClientConn": "net/http/httputil", - "httputil.DumpRequest": "net/http/httputil", - "httputil.DumpRequestOut": "net/http/httputil", - "httputil.DumpResponse": "net/http/httputil", - "httputil.ErrClosed": "net/http/httputil", - "httputil.ErrLineTooLong": "net/http/httputil", - "httputil.ErrPersistEOF": "net/http/httputil", - "httputil.ErrPipeline": "net/http/httputil", - "httputil.NewChunkedReader": "net/http/httputil", - "httputil.NewChunkedWriter": "net/http/httputil", - "httputil.NewClientConn": "net/http/httputil", - "httputil.NewProxyClientConn": "net/http/httputil", - "httputil.NewServerConn": "net/http/httputil", - "httputil.NewSingleHostReverseProxy": "net/http/httputil", - "httputil.ReverseProxy": "net/http/httputil", - "httputil.ServerConn": "net/http/httputil", - "image.Alpha": "image", - "image.Alpha16": "image", - "image.Black": "image", - "image.CMYK": "image", - "image.Config": "image", - "image.Decode": "image", - "image.DecodeConfig": "image", - "image.ErrFormat": "image", - "image.Gray": "image", - "image.Gray16": "image", - "image.Image": "image", - "image.NRGBA": "image", - "image.NRGBA64": "image", - "image.NYCbCrA": "image", - "image.NewAlpha": "image", - "image.NewAlpha16": "image", - "image.NewCMYK": "image", - "image.NewGray": "image", - "image.NewGray16": "image", - "image.NewNRGBA": "image", - "image.NewNRGBA64": "image", - "image.NewNYCbCrA": "image", - "image.NewPaletted": "image", - "image.NewRGBA": "image", - "image.NewRGBA64": "image", - "image.NewUniform": "image", - "image.NewYCbCr": "image", - "image.Opaque": "image", - "image.Paletted": "image", - "image.PalettedImage": "image", - "image.Point": "image", - "image.Pt": "image", - "image.RGBA": "image", - "image.RGBA64": "image", - "image.Rect": "image", - "image.Rectangle": "image", - "image.RegisterFormat": "image", - "image.Transparent": "image", - "image.Uniform": "image", - "image.White": "image", - "image.YCbCr": "image", - "image.YCbCrSubsampleRatio": "image", - "image.YCbCrSubsampleRatio410": "image", - "image.YCbCrSubsampleRatio411": "image", - "image.YCbCrSubsampleRatio420": "image", - "image.YCbCrSubsampleRatio422": "image", - "image.YCbCrSubsampleRatio440": "image", - "image.YCbCrSubsampleRatio444": "image", - "image.ZP": "image", - "image.ZR": "image", - "importer.Default": "go/importer", - "importer.For": "go/importer", - "importer.Lookup": "go/importer", - "io.ByteReader": "io", - "io.ByteScanner": "io", - "io.ByteWriter": "io", - "io.Closer": "io", - "io.Copy": "io", - "io.CopyBuffer": "io", - "io.CopyN": "io", - "io.EOF": "io", - "io.ErrClosedPipe": "io", - "io.ErrNoProgress": "io", - "io.ErrShortBuffer": "io", - "io.ErrShortWrite": "io", - "io.ErrUnexpectedEOF": "io", - "io.LimitReader": "io", - "io.LimitedReader": "io", - "io.MultiReader": "io", - "io.MultiWriter": "io", - "io.NewSectionReader": "io", - "io.Pipe": "io", - "io.PipeReader": "io", - "io.PipeWriter": "io", - "io.ReadAtLeast": "io", - "io.ReadCloser": "io", - "io.ReadFull": "io", - "io.ReadSeeker": "io", - "io.ReadWriteCloser": "io", - "io.ReadWriteSeeker": "io", - "io.ReadWriter": "io", - "io.Reader": "io", - "io.ReaderAt": "io", - "io.ReaderFrom": "io", - "io.RuneReader": "io", - "io.RuneScanner": "io", - "io.SectionReader": "io", - "io.SeekCurrent": "io", - "io.SeekEnd": "io", - "io.SeekStart": "io", - "io.Seeker": "io", - "io.TeeReader": "io", - "io.WriteCloser": "io", - "io.WriteSeeker": "io", - "io.WriteString": "io", - "io.Writer": "io", - "io.WriterAt": "io", - "io.WriterTo": "io", - "iotest.DataErrReader": "testing/iotest", - "iotest.ErrTimeout": "testing/iotest", - "iotest.HalfReader": "testing/iotest", - "iotest.NewReadLogger": "testing/iotest", - "iotest.NewWriteLogger": "testing/iotest", - "iotest.OneByteReader": "testing/iotest", - "iotest.TimeoutReader": "testing/iotest", - "iotest.TruncateWriter": "testing/iotest", - "ioutil.Discard": "io/ioutil", - "ioutil.NopCloser": "io/ioutil", - "ioutil.ReadAll": "io/ioutil", - "ioutil.ReadDir": "io/ioutil", - "ioutil.ReadFile": "io/ioutil", - "ioutil.TempDir": "io/ioutil", - "ioutil.TempFile": "io/ioutil", - "ioutil.WriteFile": "io/ioutil", - "jpeg.Decode": "image/jpeg", - "jpeg.DecodeConfig": "image/jpeg", - "jpeg.DefaultQuality": "image/jpeg", - "jpeg.Encode": "image/jpeg", - "jpeg.FormatError": "image/jpeg", - "jpeg.Options": "image/jpeg", - "jpeg.Reader": "image/jpeg", - "jpeg.UnsupportedError": "image/jpeg", - "json.Compact": "encoding/json", - "json.Decoder": "encoding/json", - "json.Delim": "encoding/json", - "json.Encoder": "encoding/json", - "json.HTMLEscape": "encoding/json", - "json.Indent": "encoding/json", - "json.InvalidUTF8Error": "encoding/json", - "json.InvalidUnmarshalError": "encoding/json", - "json.Marshal": "encoding/json", - "json.MarshalIndent": "encoding/json", - "json.Marshaler": "encoding/json", - "json.MarshalerError": "encoding/json", - "json.NewDecoder": "encoding/json", - "json.NewEncoder": "encoding/json", - "json.Number": "encoding/json", - "json.RawMessage": "encoding/json", - "json.SyntaxError": "encoding/json", - "json.Token": "encoding/json", - "json.Unmarshal": "encoding/json", - "json.UnmarshalFieldError": "encoding/json", - "json.UnmarshalTypeError": "encoding/json", - "json.Unmarshaler": "encoding/json", - "json.UnsupportedTypeError": "encoding/json", - "json.UnsupportedValueError": "encoding/json", - "jsonrpc.Dial": "net/rpc/jsonrpc", - "jsonrpc.NewClient": "net/rpc/jsonrpc", - "jsonrpc.NewClientCodec": "net/rpc/jsonrpc", - "jsonrpc.NewServerCodec": "net/rpc/jsonrpc", - "jsonrpc.ServeConn": "net/rpc/jsonrpc", - "list.Element": "container/list", - "list.List": "container/list", - "list.New": "container/list", - "log.Fatal": "log", - "log.Fatalf": "log", - "log.Fatalln": "log", - "log.Flags": "log", - "log.LUTC": "log", - "log.Ldate": "log", - "log.Llongfile": "log", - "log.Lmicroseconds": "log", - "log.Logger": "log", - "log.Lshortfile": "log", - "log.LstdFlags": "log", - "log.Ltime": "log", - "log.New": "log", - "log.Output": "log", - "log.Panic": "log", - "log.Panicf": "log", - "log.Panicln": "log", - "log.Prefix": "log", - "log.Print": "log", - "log.Printf": "log", - "log.Println": "log", - "log.SetFlags": "log", - "log.SetOutput": "log", - "log.SetPrefix": "log", - "lzw.LSB": "compress/lzw", - "lzw.MSB": "compress/lzw", - "lzw.NewReader": "compress/lzw", - "lzw.NewWriter": "compress/lzw", - "lzw.Order": "compress/lzw", - "macho.Cpu": "debug/macho", - "macho.Cpu386": "debug/macho", - "macho.CpuAmd64": "debug/macho", - "macho.CpuArm": "debug/macho", - "macho.CpuPpc": "debug/macho", - "macho.CpuPpc64": "debug/macho", - "macho.Dylib": "debug/macho", - "macho.DylibCmd": "debug/macho", - "macho.Dysymtab": "debug/macho", - "macho.DysymtabCmd": "debug/macho", - "macho.ErrNotFat": "debug/macho", - "macho.FatArch": "debug/macho", - "macho.FatArchHeader": "debug/macho", - "macho.FatFile": "debug/macho", - "macho.File": "debug/macho", - "macho.FileHeader": "debug/macho", - "macho.FormatError": "debug/macho", - "macho.Load": "debug/macho", - "macho.LoadBytes": "debug/macho", - "macho.LoadCmd": "debug/macho", - "macho.LoadCmdDylib": "debug/macho", - "macho.LoadCmdDylinker": "debug/macho", - "macho.LoadCmdDysymtab": "debug/macho", - "macho.LoadCmdSegment": "debug/macho", - "macho.LoadCmdSegment64": "debug/macho", - "macho.LoadCmdSymtab": "debug/macho", - "macho.LoadCmdThread": "debug/macho", - "macho.LoadCmdUnixThread": "debug/macho", - "macho.Magic32": "debug/macho", - "macho.Magic64": "debug/macho", - "macho.MagicFat": "debug/macho", - "macho.NewFatFile": "debug/macho", - "macho.NewFile": "debug/macho", - "macho.Nlist32": "debug/macho", - "macho.Nlist64": "debug/macho", - "macho.Open": "debug/macho", - "macho.OpenFat": "debug/macho", - "macho.Regs386": "debug/macho", - "macho.RegsAMD64": "debug/macho", - "macho.Section": "debug/macho", - "macho.Section32": "debug/macho", - "macho.Section64": "debug/macho", - "macho.SectionHeader": "debug/macho", - "macho.Segment": "debug/macho", - "macho.Segment32": "debug/macho", - "macho.Segment64": "debug/macho", - "macho.SegmentHeader": "debug/macho", - "macho.Symbol": "debug/macho", - "macho.Symtab": "debug/macho", - "macho.SymtabCmd": "debug/macho", - "macho.Thread": "debug/macho", - "macho.Type": "debug/macho", - "macho.TypeBundle": "debug/macho", - "macho.TypeDylib": "debug/macho", - "macho.TypeExec": "debug/macho", - "macho.TypeObj": "debug/macho", - "mail.Address": "net/mail", - "mail.AddressParser": "net/mail", - "mail.ErrHeaderNotPresent": "net/mail", - "mail.Header": "net/mail", - "mail.Message": "net/mail", - "mail.ParseAddress": "net/mail", - "mail.ParseAddressList": "net/mail", - "mail.ReadMessage": "net/mail", - "math.Abs": "math", - "math.Acos": "math", - "math.Acosh": "math", - "math.Asin": "math", - "math.Asinh": "math", - "math.Atan": "math", - "math.Atan2": "math", - "math.Atanh": "math", - "math.Cbrt": "math", - "math.Ceil": "math", - "math.Copysign": "math", - "math.Cos": "math", - "math.Cosh": "math", - "math.Dim": "math", - "math.E": "math", - "math.Erf": "math", - "math.Erfc": "math", - "math.Exp": "math", - "math.Exp2": "math", - "math.Expm1": "math", - "math.Float32bits": "math", - "math.Float32frombits": "math", - "math.Float64bits": "math", - "math.Float64frombits": "math", - "math.Floor": "math", - "math.Frexp": "math", - "math.Gamma": "math", - "math.Hypot": "math", - "math.Ilogb": "math", - "math.Inf": "math", - "math.IsInf": "math", - "math.IsNaN": "math", - "math.J0": "math", - "math.J1": "math", - "math.Jn": "math", - "math.Ldexp": "math", - "math.Lgamma": "math", - "math.Ln10": "math", - "math.Ln2": "math", - "math.Log": "math", - "math.Log10": "math", - "math.Log10E": "math", - "math.Log1p": "math", - "math.Log2": "math", - "math.Log2E": "math", - "math.Logb": "math", - "math.Max": "math", - "math.MaxFloat32": "math", - "math.MaxFloat64": "math", - "math.MaxInt16": "math", - "math.MaxInt32": "math", - "math.MaxInt64": "math", - "math.MaxInt8": "math", - "math.MaxUint16": "math", - "math.MaxUint32": "math", - "math.MaxUint64": "math", - "math.MaxUint8": "math", - "math.Min": "math", - "math.MinInt16": "math", - "math.MinInt32": "math", - "math.MinInt64": "math", - "math.MinInt8": "math", - "math.Mod": "math", - "math.Modf": "math", - "math.NaN": "math", - "math.Nextafter": "math", - "math.Nextafter32": "math", - "math.Phi": "math", - "math.Pi": "math", - "math.Pow": "math", - "math.Pow10": "math", - "math.Remainder": "math", - "math.Signbit": "math", - "math.Sin": "math", - "math.Sincos": "math", - "math.Sinh": "math", - "math.SmallestNonzeroFloat32": "math", - "math.SmallestNonzeroFloat64": "math", - "math.Sqrt": "math", - "math.Sqrt2": "math", - "math.SqrtE": "math", - "math.SqrtPhi": "math", - "math.SqrtPi": "math", - "math.Tan": "math", - "math.Tanh": "math", - "math.Trunc": "math", - "math.Y0": "math", - "math.Y1": "math", - "math.Yn": "math", - "md5.BlockSize": "crypto/md5", - "md5.New": "crypto/md5", - "md5.Size": "crypto/md5", - "md5.Sum": "crypto/md5", - "mime.AddExtensionType": "mime", - "mime.BEncoding": "mime", - "mime.ExtensionsByType": "mime", - "mime.FormatMediaType": "mime", - "mime.ParseMediaType": "mime", - "mime.QEncoding": "mime", - "mime.TypeByExtension": "mime", - "mime.WordDecoder": "mime", - "mime.WordEncoder": "mime", - "multipart.File": "mime/multipart", - "multipart.FileHeader": "mime/multipart", - "multipart.Form": "mime/multipart", - "multipart.NewReader": "mime/multipart", - "multipart.NewWriter": "mime/multipart", - "multipart.Part": "mime/multipart", - "multipart.Reader": "mime/multipart", - "multipart.Writer": "mime/multipart", - "net.Addr": "net", - "net.AddrError": "net", - "net.CIDRMask": "net", - "net.Conn": "net", - "net.DNSConfigError": "net", - "net.DNSError": "net", - "net.Dial": "net", - "net.DialIP": "net", - "net.DialTCP": "net", - "net.DialTimeout": "net", - "net.DialUDP": "net", - "net.DialUnix": "net", - "net.Dialer": "net", - "net.ErrWriteToConnected": "net", - "net.Error": "net", - "net.FileConn": "net", - "net.FileListener": "net", - "net.FilePacketConn": "net", - "net.FlagBroadcast": "net", - "net.FlagLoopback": "net", - "net.FlagMulticast": "net", - "net.FlagPointToPoint": "net", - "net.FlagUp": "net", - "net.Flags": "net", - "net.HardwareAddr": "net", - "net.IP": "net", - "net.IPAddr": "net", - "net.IPConn": "net", - "net.IPMask": "net", - "net.IPNet": "net", - "net.IPv4": "net", - "net.IPv4Mask": "net", - "net.IPv4allrouter": "net", - "net.IPv4allsys": "net", - "net.IPv4bcast": "net", - "net.IPv4len": "net", - "net.IPv4zero": "net", - "net.IPv6interfacelocalallnodes": "net", - "net.IPv6len": "net", - "net.IPv6linklocalallnodes": "net", - "net.IPv6linklocalallrouters": "net", - "net.IPv6loopback": "net", - "net.IPv6unspecified": "net", - "net.IPv6zero": "net", - "net.Interface": "net", - "net.InterfaceAddrs": "net", - "net.InterfaceByIndex": "net", - "net.InterfaceByName": "net", - "net.Interfaces": "net", - "net.InvalidAddrError": "net", - "net.JoinHostPort": "net", - "net.Listen": "net", - "net.ListenIP": "net", - "net.ListenMulticastUDP": "net", - "net.ListenPacket": "net", - "net.ListenTCP": "net", - "net.ListenUDP": "net", - "net.ListenUnix": "net", - "net.ListenUnixgram": "net", - "net.Listener": "net", - "net.LookupAddr": "net", - "net.LookupCNAME": "net", - "net.LookupHost": "net", - "net.LookupIP": "net", - "net.LookupMX": "net", - "net.LookupNS": "net", - "net.LookupPort": "net", - "net.LookupSRV": "net", - "net.LookupTXT": "net", - "net.MX": "net", - "net.NS": "net", - "net.OpError": "net", - "net.PacketConn": "net", - "net.ParseCIDR": "net", - "net.ParseError": "net", - "net.ParseIP": "net", - "net.ParseMAC": "net", - "net.Pipe": "net", - "net.ResolveIPAddr": "net", - "net.ResolveTCPAddr": "net", - "net.ResolveUDPAddr": "net", - "net.ResolveUnixAddr": "net", - "net.SRV": "net", - "net.SplitHostPort": "net", - "net.TCPAddr": "net", - "net.TCPConn": "net", - "net.TCPListener": "net", - "net.UDPAddr": "net", - "net.UDPConn": "net", - "net.UnixAddr": "net", - "net.UnixConn": "net", - "net.UnixListener": "net", - "net.UnknownNetworkError": "net", - "os.Args": "os", - "os.Chdir": "os", - "os.Chmod": "os", - "os.Chown": "os", - "os.Chtimes": "os", - "os.Clearenv": "os", - "os.Create": "os", - "os.DevNull": "os", - "os.Environ": "os", - "os.ErrExist": "os", - "os.ErrInvalid": "os", - "os.ErrNotExist": "os", - "os.ErrPermission": "os", - "os.Exit": "os", - "os.Expand": "os", - "os.ExpandEnv": "os", - "os.File": "os", - "os.FileInfo": "os", - "os.FileMode": "os", - "os.FindProcess": "os", - "os.Getegid": "os", - "os.Getenv": "os", - "os.Geteuid": "os", - "os.Getgid": "os", - "os.Getgroups": "os", - "os.Getpagesize": "os", - "os.Getpid": "os", - "os.Getppid": "os", - "os.Getuid": "os", - "os.Getwd": "os", - "os.Hostname": "os", - "os.Interrupt": "os", - "os.IsExist": "os", - "os.IsNotExist": "os", - "os.IsPathSeparator": "os", - "os.IsPermission": "os", - "os.Kill": "os", - "os.Lchown": "os", - "os.Link": "os", - "os.LinkError": "os", - "os.LookupEnv": "os", - "os.Lstat": "os", - "os.Mkdir": "os", - "os.MkdirAll": "os", - "os.ModeAppend": "os", - "os.ModeCharDevice": "os", - "os.ModeDevice": "os", - "os.ModeDir": "os", - "os.ModeExclusive": "os", - "os.ModeNamedPipe": "os", - "os.ModePerm": "os", - "os.ModeSetgid": "os", - "os.ModeSetuid": "os", - "os.ModeSocket": "os", - "os.ModeSticky": "os", - "os.ModeSymlink": "os", - "os.ModeTemporary": "os", - "os.ModeType": "os", - "os.NewFile": "os", - "os.NewSyscallError": "os", - "os.O_APPEND": "os", - "os.O_CREATE": "os", - "os.O_EXCL": "os", - "os.O_RDONLY": "os", - "os.O_RDWR": "os", - "os.O_SYNC": "os", - "os.O_TRUNC": "os", - "os.O_WRONLY": "os", - "os.Open": "os", - "os.OpenFile": "os", - "os.PathError": "os", - "os.PathListSeparator": "os", - "os.PathSeparator": "os", - "os.Pipe": "os", - "os.ProcAttr": "os", - "os.Process": "os", - "os.ProcessState": "os", - "os.Readlink": "os", - "os.Remove": "os", - "os.RemoveAll": "os", - "os.Rename": "os", - "os.SEEK_CUR": "os", - "os.SEEK_END": "os", - "os.SEEK_SET": "os", - "os.SameFile": "os", - "os.Setenv": "os", - "os.Signal": "os", - "os.StartProcess": "os", - "os.Stat": "os", - "os.Stderr": "os", - "os.Stdin": "os", - "os.Stdout": "os", - "os.Symlink": "os", - "os.SyscallError": "os", - "os.TempDir": "os", - "os.Truncate": "os", - "os.Unsetenv": "os", - "palette.Plan9": "image/color/palette", - "palette.WebSafe": "image/color/palette", - "parse.ActionNode": "text/template/parse", - "parse.BoolNode": "text/template/parse", - "parse.BranchNode": "text/template/parse", - "parse.ChainNode": "text/template/parse", - "parse.CommandNode": "text/template/parse", - "parse.DotNode": "text/template/parse", - "parse.FieldNode": "text/template/parse", - "parse.IdentifierNode": "text/template/parse", - "parse.IfNode": "text/template/parse", - "parse.IsEmptyTree": "text/template/parse", - "parse.ListNode": "text/template/parse", - "parse.New": "text/template/parse", - "parse.NewIdentifier": "text/template/parse", - "parse.NilNode": "text/template/parse", - "parse.Node": "text/template/parse", - "parse.NodeAction": "text/template/parse", - "parse.NodeBool": "text/template/parse", - "parse.NodeChain": "text/template/parse", - "parse.NodeCommand": "text/template/parse", - "parse.NodeDot": "text/template/parse", - "parse.NodeField": "text/template/parse", - "parse.NodeIdentifier": "text/template/parse", - "parse.NodeIf": "text/template/parse", - "parse.NodeList": "text/template/parse", - "parse.NodeNil": "text/template/parse", - "parse.NodeNumber": "text/template/parse", - "parse.NodePipe": "text/template/parse", - "parse.NodeRange": "text/template/parse", - "parse.NodeString": "text/template/parse", - "parse.NodeTemplate": "text/template/parse", - "parse.NodeText": "text/template/parse", - "parse.NodeType": "text/template/parse", - "parse.NodeVariable": "text/template/parse", - "parse.NodeWith": "text/template/parse", - "parse.NumberNode": "text/template/parse", - "parse.Parse": "text/template/parse", - "parse.PipeNode": "text/template/parse", - "parse.Pos": "text/template/parse", - "parse.RangeNode": "text/template/parse", - "parse.StringNode": "text/template/parse", - "parse.TemplateNode": "text/template/parse", - "parse.TextNode": "text/template/parse", - "parse.Tree": "text/template/parse", - "parse.VariableNode": "text/template/parse", - "parse.WithNode": "text/template/parse", - "parser.AllErrors": "go/parser", - "parser.DeclarationErrors": "go/parser", - "parser.ImportsOnly": "go/parser", - "parser.Mode": "go/parser", - "parser.PackageClauseOnly": "go/parser", - "parser.ParseComments": "go/parser", - "parser.ParseDir": "go/parser", - "parser.ParseExpr": "go/parser", - "parser.ParseExprFrom": "go/parser", - "parser.ParseFile": "go/parser", - "parser.SpuriousErrors": "go/parser", - "parser.Trace": "go/parser", - "path.Base": "path", - "path.Clean": "path", - "path.Dir": "path", - "path.ErrBadPattern": "path", - "path.Ext": "path", - "path.IsAbs": "path", - "path.Join": "path", - "path.Match": "path", - "path.Split": "path", - "pe.COFFSymbol": "debug/pe", - "pe.COFFSymbolSize": "debug/pe", - "pe.DataDirectory": "debug/pe", - "pe.File": "debug/pe", - "pe.FileHeader": "debug/pe", - "pe.FormatError": "debug/pe", - "pe.IMAGE_FILE_MACHINE_AM33": "debug/pe", - "pe.IMAGE_FILE_MACHINE_AMD64": "debug/pe", - "pe.IMAGE_FILE_MACHINE_ARM": "debug/pe", - "pe.IMAGE_FILE_MACHINE_EBC": "debug/pe", - "pe.IMAGE_FILE_MACHINE_I386": "debug/pe", - "pe.IMAGE_FILE_MACHINE_IA64": "debug/pe", - "pe.IMAGE_FILE_MACHINE_M32R": "debug/pe", - "pe.IMAGE_FILE_MACHINE_MIPS16": "debug/pe", - "pe.IMAGE_FILE_MACHINE_MIPSFPU": "debug/pe", - "pe.IMAGE_FILE_MACHINE_MIPSFPU16": "debug/pe", - "pe.IMAGE_FILE_MACHINE_POWERPC": "debug/pe", - "pe.IMAGE_FILE_MACHINE_POWERPCFP": "debug/pe", - "pe.IMAGE_FILE_MACHINE_R4000": "debug/pe", - "pe.IMAGE_FILE_MACHINE_SH3": "debug/pe", - "pe.IMAGE_FILE_MACHINE_SH3DSP": "debug/pe", - "pe.IMAGE_FILE_MACHINE_SH4": "debug/pe", - "pe.IMAGE_FILE_MACHINE_SH5": "debug/pe", - "pe.IMAGE_FILE_MACHINE_THUMB": "debug/pe", - "pe.IMAGE_FILE_MACHINE_UNKNOWN": "debug/pe", - "pe.IMAGE_FILE_MACHINE_WCEMIPSV2": "debug/pe", - "pe.ImportDirectory": "debug/pe", - "pe.NewFile": "debug/pe", - "pe.Open": "debug/pe", - "pe.OptionalHeader32": "debug/pe", - "pe.OptionalHeader64": "debug/pe", - "pe.Section": "debug/pe", - "pe.SectionHeader": "debug/pe", - "pe.SectionHeader32": "debug/pe", - "pe.Symbol": "debug/pe", - "pem.Block": "encoding/pem", - "pem.Decode": "encoding/pem", - "pem.Encode": "encoding/pem", - "pem.EncodeToMemory": "encoding/pem", - "pkix.AlgorithmIdentifier": "crypto/x509/pkix", - "pkix.AttributeTypeAndValue": "crypto/x509/pkix", - "pkix.AttributeTypeAndValueSET": "crypto/x509/pkix", - "pkix.CertificateList": "crypto/x509/pkix", - "pkix.Extension": "crypto/x509/pkix", - "pkix.Name": "crypto/x509/pkix", - "pkix.RDNSequence": "crypto/x509/pkix", - "pkix.RelativeDistinguishedNameSET": "crypto/x509/pkix", - "pkix.RevokedCertificate": "crypto/x509/pkix", - "pkix.TBSCertificateList": "crypto/x509/pkix", - "plan9obj.File": "debug/plan9obj", - "plan9obj.FileHeader": "debug/plan9obj", - "plan9obj.Magic386": "debug/plan9obj", - "plan9obj.Magic64": "debug/plan9obj", - "plan9obj.MagicAMD64": "debug/plan9obj", - "plan9obj.MagicARM": "debug/plan9obj", - "plan9obj.NewFile": "debug/plan9obj", - "plan9obj.Open": "debug/plan9obj", - "plan9obj.Section": "debug/plan9obj", - "plan9obj.SectionHeader": "debug/plan9obj", - "plan9obj.Sym": "debug/plan9obj", - "png.BestCompression": "image/png", - "png.BestSpeed": "image/png", - "png.CompressionLevel": "image/png", - "png.Decode": "image/png", - "png.DecodeConfig": "image/png", - "png.DefaultCompression": "image/png", - "png.Encode": "image/png", - "png.Encoder": "image/png", - "png.FormatError": "image/png", - "png.NoCompression": "image/png", - "png.UnsupportedError": "image/png", - "pprof.Cmdline": "net/http/pprof", - "pprof.Handler": "net/http/pprof", - "pprof.Index": "net/http/pprof", - "pprof.Lookup": "runtime/pprof", - "pprof.NewProfile": "runtime/pprof", - // "pprof.Profile" is ambiguous - "pprof.Profiles": "runtime/pprof", - "pprof.StartCPUProfile": "runtime/pprof", - "pprof.StopCPUProfile": "runtime/pprof", - "pprof.Symbol": "net/http/pprof", - "pprof.Trace": "net/http/pprof", - "pprof.WriteHeapProfile": "runtime/pprof", - "printer.CommentedNode": "go/printer", - "printer.Config": "go/printer", - "printer.Fprint": "go/printer", - "printer.Mode": "go/printer", - "printer.RawFormat": "go/printer", - "printer.SourcePos": "go/printer", - "printer.TabIndent": "go/printer", - "printer.UseSpaces": "go/printer", - "quick.Check": "testing/quick", - "quick.CheckEqual": "testing/quick", - "quick.CheckEqualError": "testing/quick", - "quick.CheckError": "testing/quick", - "quick.Config": "testing/quick", - "quick.Generator": "testing/quick", - "quick.SetupError": "testing/quick", - "quick.Value": "testing/quick", - "quotedprintable.NewReader": "mime/quotedprintable", - "quotedprintable.NewWriter": "mime/quotedprintable", - "quotedprintable.Reader": "mime/quotedprintable", - "quotedprintable.Writer": "mime/quotedprintable", - "rand.ExpFloat64": "math/rand", - "rand.Float32": "math/rand", - "rand.Float64": "math/rand", - // "rand.Int" is ambiguous - "rand.Int31": "math/rand", - "rand.Int31n": "math/rand", - "rand.Int63": "math/rand", - "rand.Int63n": "math/rand", - "rand.Intn": "math/rand", - "rand.New": "math/rand", - "rand.NewSource": "math/rand", - "rand.NewZipf": "math/rand", - "rand.NormFloat64": "math/rand", - "rand.Perm": "math/rand", - "rand.Prime": "crypto/rand", - "rand.Rand": "math/rand", - // "rand.Read" is ambiguous - "rand.Reader": "crypto/rand", - "rand.Seed": "math/rand", - "rand.Source": "math/rand", - "rand.Uint32": "math/rand", - "rand.Zipf": "math/rand", - "rc4.Cipher": "crypto/rc4", - "rc4.KeySizeError": "crypto/rc4", - "rc4.NewCipher": "crypto/rc4", - "reflect.Append": "reflect", - "reflect.AppendSlice": "reflect", - "reflect.Array": "reflect", - "reflect.ArrayOf": "reflect", - "reflect.Bool": "reflect", - "reflect.BothDir": "reflect", - "reflect.Chan": "reflect", - "reflect.ChanDir": "reflect", - "reflect.ChanOf": "reflect", - "reflect.Complex128": "reflect", - "reflect.Complex64": "reflect", - "reflect.Copy": "reflect", - "reflect.DeepEqual": "reflect", - "reflect.Float32": "reflect", - "reflect.Float64": "reflect", - "reflect.Func": "reflect", - "reflect.FuncOf": "reflect", - "reflect.Indirect": "reflect", - "reflect.Int": "reflect", - "reflect.Int16": "reflect", - "reflect.Int32": "reflect", - "reflect.Int64": "reflect", - "reflect.Int8": "reflect", - "reflect.Interface": "reflect", - "reflect.Invalid": "reflect", - "reflect.Kind": "reflect", - "reflect.MakeChan": "reflect", - "reflect.MakeFunc": "reflect", - "reflect.MakeMap": "reflect", - "reflect.MakeSlice": "reflect", - "reflect.Map": "reflect", - "reflect.MapOf": "reflect", - "reflect.Method": "reflect", - "reflect.New": "reflect", - "reflect.NewAt": "reflect", - "reflect.Ptr": "reflect", - "reflect.PtrTo": "reflect", - "reflect.RecvDir": "reflect", - "reflect.Select": "reflect", - "reflect.SelectCase": "reflect", - "reflect.SelectDefault": "reflect", - "reflect.SelectDir": "reflect", - "reflect.SelectRecv": "reflect", - "reflect.SelectSend": "reflect", - "reflect.SendDir": "reflect", - "reflect.Slice": "reflect", - "reflect.SliceHeader": "reflect", - "reflect.SliceOf": "reflect", - "reflect.String": "reflect", - "reflect.StringHeader": "reflect", - "reflect.Struct": "reflect", - "reflect.StructField": "reflect", - "reflect.StructOf": "reflect", - "reflect.StructTag": "reflect", - "reflect.TypeOf": "reflect", - "reflect.Uint": "reflect", - "reflect.Uint16": "reflect", - "reflect.Uint32": "reflect", - "reflect.Uint64": "reflect", - "reflect.Uint8": "reflect", - "reflect.Uintptr": "reflect", - "reflect.UnsafePointer": "reflect", - "reflect.Value": "reflect", - "reflect.ValueError": "reflect", - "reflect.ValueOf": "reflect", - "reflect.Zero": "reflect", - "regexp.Compile": "regexp", - "regexp.CompilePOSIX": "regexp", - "regexp.Match": "regexp", - "regexp.MatchReader": "regexp", - "regexp.MatchString": "regexp", - "regexp.MustCompile": "regexp", - "regexp.MustCompilePOSIX": "regexp", - "regexp.QuoteMeta": "regexp", - "regexp.Regexp": "regexp", - "ring.New": "container/ring", - "ring.Ring": "container/ring", - "rpc.Accept": "net/rpc", - "rpc.Call": "net/rpc", - "rpc.Client": "net/rpc", - "rpc.ClientCodec": "net/rpc", - "rpc.DefaultDebugPath": "net/rpc", - "rpc.DefaultRPCPath": "net/rpc", - "rpc.DefaultServer": "net/rpc", - "rpc.Dial": "net/rpc", - "rpc.DialHTTP": "net/rpc", - "rpc.DialHTTPPath": "net/rpc", - "rpc.ErrShutdown": "net/rpc", - "rpc.HandleHTTP": "net/rpc", - "rpc.NewClient": "net/rpc", - "rpc.NewClientWithCodec": "net/rpc", - "rpc.NewServer": "net/rpc", - "rpc.Register": "net/rpc", - "rpc.RegisterName": "net/rpc", - "rpc.Request": "net/rpc", - "rpc.Response": "net/rpc", - "rpc.ServeCodec": "net/rpc", - "rpc.ServeConn": "net/rpc", - "rpc.ServeRequest": "net/rpc", - "rpc.Server": "net/rpc", - "rpc.ServerCodec": "net/rpc", - "rpc.ServerError": "net/rpc", - "rsa.CRTValue": "crypto/rsa", - "rsa.DecryptOAEP": "crypto/rsa", - "rsa.DecryptPKCS1v15": "crypto/rsa", - "rsa.DecryptPKCS1v15SessionKey": "crypto/rsa", - "rsa.EncryptOAEP": "crypto/rsa", - "rsa.EncryptPKCS1v15": "crypto/rsa", - "rsa.ErrDecryption": "crypto/rsa", - "rsa.ErrMessageTooLong": "crypto/rsa", - "rsa.ErrVerification": "crypto/rsa", - "rsa.GenerateKey": "crypto/rsa", - "rsa.GenerateMultiPrimeKey": "crypto/rsa", - "rsa.OAEPOptions": "crypto/rsa", - "rsa.PKCS1v15DecryptOptions": "crypto/rsa", - "rsa.PSSOptions": "crypto/rsa", - "rsa.PSSSaltLengthAuto": "crypto/rsa", - "rsa.PSSSaltLengthEqualsHash": "crypto/rsa", - "rsa.PrecomputedValues": "crypto/rsa", - "rsa.PrivateKey": "crypto/rsa", - "rsa.PublicKey": "crypto/rsa", - "rsa.SignPKCS1v15": "crypto/rsa", - "rsa.SignPSS": "crypto/rsa", - "rsa.VerifyPKCS1v15": "crypto/rsa", - "rsa.VerifyPSS": "crypto/rsa", - "runtime.BlockProfile": "runtime", - "runtime.BlockProfileRecord": "runtime", - "runtime.Breakpoint": "runtime", - "runtime.CPUProfile": "runtime", - "runtime.Caller": "runtime", - "runtime.Callers": "runtime", - "runtime.CallersFrames": "runtime", - "runtime.Compiler": "runtime", - "runtime.Error": "runtime", - "runtime.Frame": "runtime", - "runtime.Frames": "runtime", - "runtime.Func": "runtime", - "runtime.FuncForPC": "runtime", - "runtime.GC": "runtime", - "runtime.GOARCH": "runtime", - "runtime.GOMAXPROCS": "runtime", - "runtime.GOOS": "runtime", - "runtime.GOROOT": "runtime", - "runtime.Goexit": "runtime", - "runtime.GoroutineProfile": "runtime", - "runtime.Gosched": "runtime", - "runtime.KeepAlive": "runtime", - "runtime.LockOSThread": "runtime", - "runtime.MemProfile": "runtime", - "runtime.MemProfileRate": "runtime", - "runtime.MemProfileRecord": "runtime", - "runtime.MemStats": "runtime", - "runtime.NumCPU": "runtime", - "runtime.NumCgoCall": "runtime", - "runtime.NumGoroutine": "runtime", - "runtime.ReadMemStats": "runtime", - "runtime.ReadTrace": "runtime", - "runtime.SetBlockProfileRate": "runtime", - "runtime.SetCPUProfileRate": "runtime", - "runtime.SetCgoTraceback": "runtime", - "runtime.SetFinalizer": "runtime", - "runtime.Stack": "runtime", - "runtime.StackRecord": "runtime", - "runtime.StartTrace": "runtime", - "runtime.StopTrace": "runtime", - "runtime.ThreadCreateProfile": "runtime", - "runtime.TypeAssertionError": "runtime", - "runtime.UnlockOSThread": "runtime", - "runtime.Version": "runtime", - "scanner.Char": "text/scanner", - "scanner.Comment": "text/scanner", - "scanner.EOF": "text/scanner", - "scanner.Error": "go/scanner", - "scanner.ErrorHandler": "go/scanner", - "scanner.ErrorList": "go/scanner", - "scanner.Float": "text/scanner", - "scanner.GoTokens": "text/scanner", - "scanner.GoWhitespace": "text/scanner", - "scanner.Ident": "text/scanner", - "scanner.Int": "text/scanner", - "scanner.Mode": "go/scanner", - "scanner.Position": "text/scanner", - "scanner.PrintError": "go/scanner", - "scanner.RawString": "text/scanner", - "scanner.ScanChars": "text/scanner", - // "scanner.ScanComments" is ambiguous - "scanner.ScanFloats": "text/scanner", - "scanner.ScanIdents": "text/scanner", - "scanner.ScanInts": "text/scanner", - "scanner.ScanRawStrings": "text/scanner", - "scanner.ScanStrings": "text/scanner", - // "scanner.Scanner" is ambiguous - "scanner.SkipComments": "text/scanner", - "scanner.String": "text/scanner", - "scanner.TokenString": "text/scanner", - "sha1.BlockSize": "crypto/sha1", - "sha1.New": "crypto/sha1", - "sha1.Size": "crypto/sha1", - "sha1.Sum": "crypto/sha1", - "sha256.BlockSize": "crypto/sha256", - "sha256.New": "crypto/sha256", - "sha256.New224": "crypto/sha256", - "sha256.Size": "crypto/sha256", - "sha256.Size224": "crypto/sha256", - "sha256.Sum224": "crypto/sha256", - "sha256.Sum256": "crypto/sha256", - "sha512.BlockSize": "crypto/sha512", - "sha512.New": "crypto/sha512", - "sha512.New384": "crypto/sha512", - "sha512.New512_224": "crypto/sha512", - "sha512.New512_256": "crypto/sha512", - "sha512.Size": "crypto/sha512", - "sha512.Size224": "crypto/sha512", - "sha512.Size256": "crypto/sha512", - "sha512.Size384": "crypto/sha512", - "sha512.Sum384": "crypto/sha512", - "sha512.Sum512": "crypto/sha512", - "sha512.Sum512_224": "crypto/sha512", - "sha512.Sum512_256": "crypto/sha512", - "signal.Ignore": "os/signal", - "signal.Notify": "os/signal", - "signal.Reset": "os/signal", - "signal.Stop": "os/signal", - "smtp.Auth": "net/smtp", - "smtp.CRAMMD5Auth": "net/smtp", - "smtp.Client": "net/smtp", - "smtp.Dial": "net/smtp", - "smtp.NewClient": "net/smtp", - "smtp.PlainAuth": "net/smtp", - "smtp.SendMail": "net/smtp", - "smtp.ServerInfo": "net/smtp", - "sort.Float64Slice": "sort", - "sort.Float64s": "sort", - "sort.Float64sAreSorted": "sort", - "sort.IntSlice": "sort", - "sort.Interface": "sort", - "sort.Ints": "sort", - "sort.IntsAreSorted": "sort", - "sort.IsSorted": "sort", - "sort.Reverse": "sort", - "sort.Search": "sort", - "sort.SearchFloat64s": "sort", - "sort.SearchInts": "sort", - "sort.SearchStrings": "sort", - "sort.Sort": "sort", - "sort.Stable": "sort", - "sort.StringSlice": "sort", - "sort.Strings": "sort", - "sort.StringsAreSorted": "sort", - "sql.DB": "database/sql", - "sql.DBStats": "database/sql", - "sql.Drivers": "database/sql", - "sql.ErrNoRows": "database/sql", - "sql.ErrTxDone": "database/sql", - "sql.NullBool": "database/sql", - "sql.NullFloat64": "database/sql", - "sql.NullInt64": "database/sql", - "sql.NullString": "database/sql", - "sql.Open": "database/sql", - "sql.RawBytes": "database/sql", - "sql.Register": "database/sql", - "sql.Result": "database/sql", - "sql.Row": "database/sql", - "sql.Rows": "database/sql", - "sql.Scanner": "database/sql", - "sql.Stmt": "database/sql", - "sql.Tx": "database/sql", - "strconv.AppendBool": "strconv", - "strconv.AppendFloat": "strconv", - "strconv.AppendInt": "strconv", - "strconv.AppendQuote": "strconv", - "strconv.AppendQuoteRune": "strconv", - "strconv.AppendQuoteRuneToASCII": "strconv", - "strconv.AppendQuoteRuneToGraphic": "strconv", - "strconv.AppendQuoteToASCII": "strconv", - "strconv.AppendQuoteToGraphic": "strconv", - "strconv.AppendUint": "strconv", - "strconv.Atoi": "strconv", - "strconv.CanBackquote": "strconv", - "strconv.ErrRange": "strconv", - "strconv.ErrSyntax": "strconv", - "strconv.FormatBool": "strconv", - "strconv.FormatFloat": "strconv", - "strconv.FormatInt": "strconv", - "strconv.FormatUint": "strconv", - "strconv.IntSize": "strconv", - "strconv.IsGraphic": "strconv", - "strconv.IsPrint": "strconv", - "strconv.Itoa": "strconv", - "strconv.NumError": "strconv", - "strconv.ParseBool": "strconv", - "strconv.ParseFloat": "strconv", - "strconv.ParseInt": "strconv", - "strconv.ParseUint": "strconv", - "strconv.Quote": "strconv", - "strconv.QuoteRune": "strconv", - "strconv.QuoteRuneToASCII": "strconv", - "strconv.QuoteRuneToGraphic": "strconv", - "strconv.QuoteToASCII": "strconv", - "strconv.QuoteToGraphic": "strconv", - "strconv.Unquote": "strconv", - "strconv.UnquoteChar": "strconv", - "strings.Compare": "strings", - "strings.Contains": "strings", - "strings.ContainsAny": "strings", - "strings.ContainsRune": "strings", - "strings.Count": "strings", - "strings.EqualFold": "strings", - "strings.Fields": "strings", - "strings.FieldsFunc": "strings", - "strings.HasPrefix": "strings", - "strings.HasSuffix": "strings", - "strings.Index": "strings", - "strings.IndexAny": "strings", - "strings.IndexByte": "strings", - "strings.IndexFunc": "strings", - "strings.IndexRune": "strings", - "strings.Join": "strings", - "strings.LastIndex": "strings", - "strings.LastIndexAny": "strings", - "strings.LastIndexByte": "strings", - "strings.LastIndexFunc": "strings", - "strings.Map": "strings", - "strings.NewReader": "strings", - "strings.NewReplacer": "strings", - "strings.Reader": "strings", - "strings.Repeat": "strings", - "strings.Replace": "strings", - "strings.Replacer": "strings", - "strings.Split": "strings", - "strings.SplitAfter": "strings", - "strings.SplitAfterN": "strings", - "strings.SplitN": "strings", - "strings.Title": "strings", - "strings.ToLower": "strings", - "strings.ToLowerSpecial": "strings", - "strings.ToTitle": "strings", - "strings.ToTitleSpecial": "strings", - "strings.ToUpper": "strings", - "strings.ToUpperSpecial": "strings", - "strings.Trim": "strings", - "strings.TrimFunc": "strings", - "strings.TrimLeft": "strings", - "strings.TrimLeftFunc": "strings", - "strings.TrimPrefix": "strings", - "strings.TrimRight": "strings", - "strings.TrimRightFunc": "strings", - "strings.TrimSpace": "strings", - "strings.TrimSuffix": "strings", - "subtle.ConstantTimeByteEq": "crypto/subtle", - "subtle.ConstantTimeCompare": "crypto/subtle", - "subtle.ConstantTimeCopy": "crypto/subtle", - "subtle.ConstantTimeEq": "crypto/subtle", - "subtle.ConstantTimeLessOrEq": "crypto/subtle", - "subtle.ConstantTimeSelect": "crypto/subtle", - "suffixarray.Index": "index/suffixarray", - "suffixarray.New": "index/suffixarray", - "sync.Cond": "sync", - "sync.Locker": "sync", - "sync.Mutex": "sync", - "sync.NewCond": "sync", - "sync.Once": "sync", - "sync.Pool": "sync", - "sync.RWMutex": "sync", - "sync.WaitGroup": "sync", - "syntax.ClassNL": "regexp/syntax", - "syntax.Compile": "regexp/syntax", - "syntax.DotNL": "regexp/syntax", - "syntax.EmptyBeginLine": "regexp/syntax", - "syntax.EmptyBeginText": "regexp/syntax", - "syntax.EmptyEndLine": "regexp/syntax", - "syntax.EmptyEndText": "regexp/syntax", - "syntax.EmptyNoWordBoundary": "regexp/syntax", - "syntax.EmptyOp": "regexp/syntax", - "syntax.EmptyOpContext": "regexp/syntax", - "syntax.EmptyWordBoundary": "regexp/syntax", - "syntax.ErrInternalError": "regexp/syntax", - "syntax.ErrInvalidCharClass": "regexp/syntax", - "syntax.ErrInvalidCharRange": "regexp/syntax", - "syntax.ErrInvalidEscape": "regexp/syntax", - "syntax.ErrInvalidNamedCapture": "regexp/syntax", - "syntax.ErrInvalidPerlOp": "regexp/syntax", - "syntax.ErrInvalidRepeatOp": "regexp/syntax", - "syntax.ErrInvalidRepeatSize": "regexp/syntax", - "syntax.ErrInvalidUTF8": "regexp/syntax", - "syntax.ErrMissingBracket": "regexp/syntax", - "syntax.ErrMissingParen": "regexp/syntax", - "syntax.ErrMissingRepeatArgument": "regexp/syntax", - "syntax.ErrTrailingBackslash": "regexp/syntax", - "syntax.ErrUnexpectedParen": "regexp/syntax", - "syntax.Error": "regexp/syntax", - "syntax.ErrorCode": "regexp/syntax", - "syntax.Flags": "regexp/syntax", - "syntax.FoldCase": "regexp/syntax", - "syntax.Inst": "regexp/syntax", - "syntax.InstAlt": "regexp/syntax", - "syntax.InstAltMatch": "regexp/syntax", - "syntax.InstCapture": "regexp/syntax", - "syntax.InstEmptyWidth": "regexp/syntax", - "syntax.InstFail": "regexp/syntax", - "syntax.InstMatch": "regexp/syntax", - "syntax.InstNop": "regexp/syntax", - "syntax.InstOp": "regexp/syntax", - "syntax.InstRune": "regexp/syntax", - "syntax.InstRune1": "regexp/syntax", - "syntax.InstRuneAny": "regexp/syntax", - "syntax.InstRuneAnyNotNL": "regexp/syntax", - "syntax.IsWordChar": "regexp/syntax", - "syntax.Literal": "regexp/syntax", - "syntax.MatchNL": "regexp/syntax", - "syntax.NonGreedy": "regexp/syntax", - "syntax.OneLine": "regexp/syntax", - "syntax.Op": "regexp/syntax", - "syntax.OpAlternate": "regexp/syntax", - "syntax.OpAnyChar": "regexp/syntax", - "syntax.OpAnyCharNotNL": "regexp/syntax", - "syntax.OpBeginLine": "regexp/syntax", - "syntax.OpBeginText": "regexp/syntax", - "syntax.OpCapture": "regexp/syntax", - "syntax.OpCharClass": "regexp/syntax", - "syntax.OpConcat": "regexp/syntax", - "syntax.OpEmptyMatch": "regexp/syntax", - "syntax.OpEndLine": "regexp/syntax", - "syntax.OpEndText": "regexp/syntax", - "syntax.OpLiteral": "regexp/syntax", - "syntax.OpNoMatch": "regexp/syntax", - "syntax.OpNoWordBoundary": "regexp/syntax", - "syntax.OpPlus": "regexp/syntax", - "syntax.OpQuest": "regexp/syntax", - "syntax.OpRepeat": "regexp/syntax", - "syntax.OpStar": "regexp/syntax", - "syntax.OpWordBoundary": "regexp/syntax", - "syntax.POSIX": "regexp/syntax", - "syntax.Parse": "regexp/syntax", - "syntax.Perl": "regexp/syntax", - "syntax.PerlX": "regexp/syntax", - "syntax.Prog": "regexp/syntax", - "syntax.Regexp": "regexp/syntax", - "syntax.Simple": "regexp/syntax", - "syntax.UnicodeGroups": "regexp/syntax", - "syntax.WasDollar": "regexp/syntax", - "syscall.AF_ALG": "syscall", - "syscall.AF_APPLETALK": "syscall", - "syscall.AF_ARP": "syscall", - "syscall.AF_ASH": "syscall", - "syscall.AF_ATM": "syscall", - "syscall.AF_ATMPVC": "syscall", - "syscall.AF_ATMSVC": "syscall", - "syscall.AF_AX25": "syscall", - "syscall.AF_BLUETOOTH": "syscall", - "syscall.AF_BRIDGE": "syscall", - "syscall.AF_CAIF": "syscall", - "syscall.AF_CAN": "syscall", - "syscall.AF_CCITT": "syscall", - "syscall.AF_CHAOS": "syscall", - "syscall.AF_CNT": "syscall", - "syscall.AF_COIP": "syscall", - "syscall.AF_DATAKIT": "syscall", - "syscall.AF_DECnet": "syscall", - "syscall.AF_DLI": "syscall", - "syscall.AF_E164": "syscall", - "syscall.AF_ECMA": "syscall", - "syscall.AF_ECONET": "syscall", - "syscall.AF_ENCAP": "syscall", - "syscall.AF_FILE": "syscall", - "syscall.AF_HYLINK": "syscall", - "syscall.AF_IEEE80211": "syscall", - "syscall.AF_IEEE802154": "syscall", - "syscall.AF_IMPLINK": "syscall", - "syscall.AF_INET": "syscall", - "syscall.AF_INET6": "syscall", - "syscall.AF_INET6_SDP": "syscall", - "syscall.AF_INET_SDP": "syscall", - "syscall.AF_IPX": "syscall", - "syscall.AF_IRDA": "syscall", - "syscall.AF_ISDN": "syscall", - "syscall.AF_ISO": "syscall", - "syscall.AF_IUCV": "syscall", - "syscall.AF_KEY": "syscall", - "syscall.AF_LAT": "syscall", - "syscall.AF_LINK": "syscall", - "syscall.AF_LLC": "syscall", - "syscall.AF_LOCAL": "syscall", - "syscall.AF_MAX": "syscall", - "syscall.AF_MPLS": "syscall", - "syscall.AF_NATM": "syscall", - "syscall.AF_NDRV": "syscall", - "syscall.AF_NETBEUI": "syscall", - "syscall.AF_NETBIOS": "syscall", - "syscall.AF_NETGRAPH": "syscall", - "syscall.AF_NETLINK": "syscall", - "syscall.AF_NETROM": "syscall", - "syscall.AF_NS": "syscall", - "syscall.AF_OROUTE": "syscall", - "syscall.AF_OSI": "syscall", - "syscall.AF_PACKET": "syscall", - "syscall.AF_PHONET": "syscall", - "syscall.AF_PPP": "syscall", - "syscall.AF_PPPOX": "syscall", - "syscall.AF_PUP": "syscall", - "syscall.AF_RDS": "syscall", - "syscall.AF_RESERVED_36": "syscall", - "syscall.AF_ROSE": "syscall", - "syscall.AF_ROUTE": "syscall", - "syscall.AF_RXRPC": "syscall", - "syscall.AF_SCLUSTER": "syscall", - "syscall.AF_SECURITY": "syscall", - "syscall.AF_SIP": "syscall", - "syscall.AF_SLOW": "syscall", - "syscall.AF_SNA": "syscall", - "syscall.AF_SYSTEM": "syscall", - "syscall.AF_TIPC": "syscall", - "syscall.AF_UNIX": "syscall", - "syscall.AF_UNSPEC": "syscall", - "syscall.AF_VENDOR00": "syscall", - "syscall.AF_VENDOR01": "syscall", - "syscall.AF_VENDOR02": "syscall", - "syscall.AF_VENDOR03": "syscall", - "syscall.AF_VENDOR04": "syscall", - "syscall.AF_VENDOR05": "syscall", - "syscall.AF_VENDOR06": "syscall", - "syscall.AF_VENDOR07": "syscall", - "syscall.AF_VENDOR08": "syscall", - "syscall.AF_VENDOR09": "syscall", - "syscall.AF_VENDOR10": "syscall", - "syscall.AF_VENDOR11": "syscall", - "syscall.AF_VENDOR12": "syscall", - "syscall.AF_VENDOR13": "syscall", - "syscall.AF_VENDOR14": "syscall", - "syscall.AF_VENDOR15": "syscall", - "syscall.AF_VENDOR16": "syscall", - "syscall.AF_VENDOR17": "syscall", - "syscall.AF_VENDOR18": "syscall", - "syscall.AF_VENDOR19": "syscall", - "syscall.AF_VENDOR20": "syscall", - "syscall.AF_VENDOR21": "syscall", - "syscall.AF_VENDOR22": "syscall", - "syscall.AF_VENDOR23": "syscall", - "syscall.AF_VENDOR24": "syscall", - "syscall.AF_VENDOR25": "syscall", - "syscall.AF_VENDOR26": "syscall", - "syscall.AF_VENDOR27": "syscall", - "syscall.AF_VENDOR28": "syscall", - "syscall.AF_VENDOR29": "syscall", - "syscall.AF_VENDOR30": "syscall", - "syscall.AF_VENDOR31": "syscall", - "syscall.AF_VENDOR32": "syscall", - "syscall.AF_VENDOR33": "syscall", - "syscall.AF_VENDOR34": "syscall", - "syscall.AF_VENDOR35": "syscall", - "syscall.AF_VENDOR36": "syscall", - "syscall.AF_VENDOR37": "syscall", - "syscall.AF_VENDOR38": "syscall", - "syscall.AF_VENDOR39": "syscall", - "syscall.AF_VENDOR40": "syscall", - "syscall.AF_VENDOR41": "syscall", - "syscall.AF_VENDOR42": "syscall", - "syscall.AF_VENDOR43": "syscall", - "syscall.AF_VENDOR44": "syscall", - "syscall.AF_VENDOR45": "syscall", - "syscall.AF_VENDOR46": "syscall", - "syscall.AF_VENDOR47": "syscall", - "syscall.AF_WANPIPE": "syscall", - "syscall.AF_X25": "syscall", - "syscall.AI_CANONNAME": "syscall", - "syscall.AI_NUMERICHOST": "syscall", - "syscall.AI_PASSIVE": "syscall", - "syscall.APPLICATION_ERROR": "syscall", - "syscall.ARPHRD_ADAPT": "syscall", - "syscall.ARPHRD_APPLETLK": "syscall", - "syscall.ARPHRD_ARCNET": "syscall", - "syscall.ARPHRD_ASH": "syscall", - "syscall.ARPHRD_ATM": "syscall", - "syscall.ARPHRD_AX25": "syscall", - "syscall.ARPHRD_BIF": "syscall", - "syscall.ARPHRD_CHAOS": "syscall", - "syscall.ARPHRD_CISCO": "syscall", - "syscall.ARPHRD_CSLIP": "syscall", - "syscall.ARPHRD_CSLIP6": "syscall", - "syscall.ARPHRD_DDCMP": "syscall", - "syscall.ARPHRD_DLCI": "syscall", - "syscall.ARPHRD_ECONET": "syscall", - "syscall.ARPHRD_EETHER": "syscall", - "syscall.ARPHRD_ETHER": "syscall", - "syscall.ARPHRD_EUI64": "syscall", - "syscall.ARPHRD_FCAL": "syscall", - "syscall.ARPHRD_FCFABRIC": "syscall", - "syscall.ARPHRD_FCPL": "syscall", - "syscall.ARPHRD_FCPP": "syscall", - "syscall.ARPHRD_FDDI": "syscall", - "syscall.ARPHRD_FRAD": "syscall", - "syscall.ARPHRD_FRELAY": "syscall", - "syscall.ARPHRD_HDLC": "syscall", - "syscall.ARPHRD_HIPPI": "syscall", - "syscall.ARPHRD_HWX25": "syscall", - "syscall.ARPHRD_IEEE1394": "syscall", - "syscall.ARPHRD_IEEE802": "syscall", - "syscall.ARPHRD_IEEE80211": "syscall", - "syscall.ARPHRD_IEEE80211_PRISM": "syscall", - "syscall.ARPHRD_IEEE80211_RADIOTAP": "syscall", - "syscall.ARPHRD_IEEE802154": "syscall", - "syscall.ARPHRD_IEEE802154_PHY": "syscall", - "syscall.ARPHRD_IEEE802_TR": "syscall", - "syscall.ARPHRD_INFINIBAND": "syscall", - "syscall.ARPHRD_IPDDP": "syscall", - "syscall.ARPHRD_IPGRE": "syscall", - "syscall.ARPHRD_IRDA": "syscall", - "syscall.ARPHRD_LAPB": "syscall", - "syscall.ARPHRD_LOCALTLK": "syscall", - "syscall.ARPHRD_LOOPBACK": "syscall", - "syscall.ARPHRD_METRICOM": "syscall", - "syscall.ARPHRD_NETROM": "syscall", - "syscall.ARPHRD_NONE": "syscall", - "syscall.ARPHRD_PIMREG": "syscall", - "syscall.ARPHRD_PPP": "syscall", - "syscall.ARPHRD_PRONET": "syscall", - "syscall.ARPHRD_RAWHDLC": "syscall", - "syscall.ARPHRD_ROSE": "syscall", - "syscall.ARPHRD_RSRVD": "syscall", - "syscall.ARPHRD_SIT": "syscall", - "syscall.ARPHRD_SKIP": "syscall", - "syscall.ARPHRD_SLIP": "syscall", - "syscall.ARPHRD_SLIP6": "syscall", - "syscall.ARPHRD_STRIP": "syscall", - "syscall.ARPHRD_TUNNEL": "syscall", - "syscall.ARPHRD_TUNNEL6": "syscall", - "syscall.ARPHRD_VOID": "syscall", - "syscall.ARPHRD_X25": "syscall", - "syscall.AUTHTYPE_CLIENT": "syscall", - "syscall.AUTHTYPE_SERVER": "syscall", - "syscall.Accept": "syscall", - "syscall.Accept4": "syscall", - "syscall.AcceptEx": "syscall", - "syscall.Access": "syscall", - "syscall.Acct": "syscall", - "syscall.AddrinfoW": "syscall", - "syscall.Adjtime": "syscall", - "syscall.Adjtimex": "syscall", - "syscall.AttachLsf": "syscall", - "syscall.B0": "syscall", - "syscall.B1000000": "syscall", - "syscall.B110": "syscall", - "syscall.B115200": "syscall", - "syscall.B1152000": "syscall", - "syscall.B1200": "syscall", - "syscall.B134": "syscall", - "syscall.B14400": "syscall", - "syscall.B150": "syscall", - "syscall.B1500000": "syscall", - "syscall.B1800": "syscall", - "syscall.B19200": "syscall", - "syscall.B200": "syscall", - "syscall.B2000000": "syscall", - "syscall.B230400": "syscall", - "syscall.B2400": "syscall", - "syscall.B2500000": "syscall", - "syscall.B28800": "syscall", - "syscall.B300": "syscall", - "syscall.B3000000": "syscall", - "syscall.B3500000": "syscall", - "syscall.B38400": "syscall", - "syscall.B4000000": "syscall", - "syscall.B460800": "syscall", - "syscall.B4800": "syscall", - "syscall.B50": "syscall", - "syscall.B500000": "syscall", - "syscall.B57600": "syscall", - "syscall.B576000": "syscall", - "syscall.B600": "syscall", - "syscall.B7200": "syscall", - "syscall.B75": "syscall", - "syscall.B76800": "syscall", - "syscall.B921600": "syscall", - "syscall.B9600": "syscall", - "syscall.BASE_PROTOCOL": "syscall", - "syscall.BIOCFEEDBACK": "syscall", - "syscall.BIOCFLUSH": "syscall", - "syscall.BIOCGBLEN": "syscall", - "syscall.BIOCGDIRECTION": "syscall", - "syscall.BIOCGDIRFILT": "syscall", - "syscall.BIOCGDLT": "syscall", - "syscall.BIOCGDLTLIST": "syscall", - "syscall.BIOCGETBUFMODE": "syscall", - "syscall.BIOCGETIF": "syscall", - "syscall.BIOCGETZMAX": "syscall", - "syscall.BIOCGFEEDBACK": "syscall", - "syscall.BIOCGFILDROP": "syscall", - "syscall.BIOCGHDRCMPLT": "syscall", - "syscall.BIOCGRSIG": "syscall", - "syscall.BIOCGRTIMEOUT": "syscall", - "syscall.BIOCGSEESENT": "syscall", - "syscall.BIOCGSTATS": "syscall", - "syscall.BIOCGSTATSOLD": "syscall", - "syscall.BIOCGTSTAMP": "syscall", - "syscall.BIOCIMMEDIATE": "syscall", - "syscall.BIOCLOCK": "syscall", - "syscall.BIOCPROMISC": "syscall", - "syscall.BIOCROTZBUF": "syscall", - "syscall.BIOCSBLEN": "syscall", - "syscall.BIOCSDIRECTION": "syscall", - "syscall.BIOCSDIRFILT": "syscall", - "syscall.BIOCSDLT": "syscall", - "syscall.BIOCSETBUFMODE": "syscall", - "syscall.BIOCSETF": "syscall", - "syscall.BIOCSETFNR": "syscall", - "syscall.BIOCSETIF": "syscall", - "syscall.BIOCSETWF": "syscall", - "syscall.BIOCSETZBUF": "syscall", - "syscall.BIOCSFEEDBACK": "syscall", - "syscall.BIOCSFILDROP": "syscall", - "syscall.BIOCSHDRCMPLT": "syscall", - "syscall.BIOCSRSIG": "syscall", - "syscall.BIOCSRTIMEOUT": "syscall", - "syscall.BIOCSSEESENT": "syscall", - "syscall.BIOCSTCPF": "syscall", - "syscall.BIOCSTSTAMP": "syscall", - "syscall.BIOCSUDPF": "syscall", - "syscall.BIOCVERSION": "syscall", - "syscall.BPF_A": "syscall", - "syscall.BPF_ABS": "syscall", - "syscall.BPF_ADD": "syscall", - "syscall.BPF_ALIGNMENT": "syscall", - "syscall.BPF_ALIGNMENT32": "syscall", - "syscall.BPF_ALU": "syscall", - "syscall.BPF_AND": "syscall", - "syscall.BPF_B": "syscall", - "syscall.BPF_BUFMODE_BUFFER": "syscall", - "syscall.BPF_BUFMODE_ZBUF": "syscall", - "syscall.BPF_DFLTBUFSIZE": "syscall", - "syscall.BPF_DIRECTION_IN": "syscall", - "syscall.BPF_DIRECTION_OUT": "syscall", - "syscall.BPF_DIV": "syscall", - "syscall.BPF_H": "syscall", - "syscall.BPF_IMM": "syscall", - "syscall.BPF_IND": "syscall", - "syscall.BPF_JA": "syscall", - "syscall.BPF_JEQ": "syscall", - "syscall.BPF_JGE": "syscall", - "syscall.BPF_JGT": "syscall", - "syscall.BPF_JMP": "syscall", - "syscall.BPF_JSET": "syscall", - "syscall.BPF_K": "syscall", - "syscall.BPF_LD": "syscall", - "syscall.BPF_LDX": "syscall", - "syscall.BPF_LEN": "syscall", - "syscall.BPF_LSH": "syscall", - "syscall.BPF_MAJOR_VERSION": "syscall", - "syscall.BPF_MAXBUFSIZE": "syscall", - "syscall.BPF_MAXINSNS": "syscall", - "syscall.BPF_MEM": "syscall", - "syscall.BPF_MEMWORDS": "syscall", - "syscall.BPF_MINBUFSIZE": "syscall", - "syscall.BPF_MINOR_VERSION": "syscall", - "syscall.BPF_MISC": "syscall", - "syscall.BPF_MSH": "syscall", - "syscall.BPF_MUL": "syscall", - "syscall.BPF_NEG": "syscall", - "syscall.BPF_OR": "syscall", - "syscall.BPF_RELEASE": "syscall", - "syscall.BPF_RET": "syscall", - "syscall.BPF_RSH": "syscall", - "syscall.BPF_ST": "syscall", - "syscall.BPF_STX": "syscall", - "syscall.BPF_SUB": "syscall", - "syscall.BPF_TAX": "syscall", - "syscall.BPF_TXA": "syscall", - "syscall.BPF_T_BINTIME": "syscall", - "syscall.BPF_T_BINTIME_FAST": "syscall", - "syscall.BPF_T_BINTIME_MONOTONIC": "syscall", - "syscall.BPF_T_BINTIME_MONOTONIC_FAST": "syscall", - "syscall.BPF_T_FAST": "syscall", - "syscall.BPF_T_FLAG_MASK": "syscall", - "syscall.BPF_T_FORMAT_MASK": "syscall", - "syscall.BPF_T_MICROTIME": "syscall", - "syscall.BPF_T_MICROTIME_FAST": "syscall", - "syscall.BPF_T_MICROTIME_MONOTONIC": "syscall", - "syscall.BPF_T_MICROTIME_MONOTONIC_FAST": "syscall", - "syscall.BPF_T_MONOTONIC": "syscall", - "syscall.BPF_T_MONOTONIC_FAST": "syscall", - "syscall.BPF_T_NANOTIME": "syscall", - "syscall.BPF_T_NANOTIME_FAST": "syscall", - "syscall.BPF_T_NANOTIME_MONOTONIC": "syscall", - "syscall.BPF_T_NANOTIME_MONOTONIC_FAST": "syscall", - "syscall.BPF_T_NONE": "syscall", - "syscall.BPF_T_NORMAL": "syscall", - "syscall.BPF_W": "syscall", - "syscall.BPF_X": "syscall", - "syscall.BRKINT": "syscall", - "syscall.Bind": "syscall", - "syscall.BindToDevice": "syscall", - "syscall.BpfBuflen": "syscall", - "syscall.BpfDatalink": "syscall", - "syscall.BpfHdr": "syscall", - "syscall.BpfHeadercmpl": "syscall", - "syscall.BpfInsn": "syscall", - "syscall.BpfInterface": "syscall", - "syscall.BpfJump": "syscall", - "syscall.BpfProgram": "syscall", - "syscall.BpfStat": "syscall", - "syscall.BpfStats": "syscall", - "syscall.BpfStmt": "syscall", - "syscall.BpfTimeout": "syscall", - "syscall.BpfTimeval": "syscall", - "syscall.BpfVersion": "syscall", - "syscall.BpfZbuf": "syscall", - "syscall.BpfZbufHeader": "syscall", - "syscall.ByHandleFileInformation": "syscall", - "syscall.BytePtrFromString": "syscall", - "syscall.ByteSliceFromString": "syscall", - "syscall.CCR0_FLUSH": "syscall", - "syscall.CERT_CHAIN_POLICY_AUTHENTICODE": "syscall", - "syscall.CERT_CHAIN_POLICY_AUTHENTICODE_TS": "syscall", - "syscall.CERT_CHAIN_POLICY_BASE": "syscall", - "syscall.CERT_CHAIN_POLICY_BASIC_CONSTRAINTS": "syscall", - "syscall.CERT_CHAIN_POLICY_EV": "syscall", - "syscall.CERT_CHAIN_POLICY_MICROSOFT_ROOT": "syscall", - "syscall.CERT_CHAIN_POLICY_NT_AUTH": "syscall", - "syscall.CERT_CHAIN_POLICY_SSL": "syscall", - "syscall.CERT_E_CN_NO_MATCH": "syscall", - "syscall.CERT_E_EXPIRED": "syscall", - "syscall.CERT_E_PURPOSE": "syscall", - "syscall.CERT_E_ROLE": "syscall", - "syscall.CERT_E_UNTRUSTEDROOT": "syscall", - "syscall.CERT_STORE_ADD_ALWAYS": "syscall", - "syscall.CERT_STORE_DEFER_CLOSE_UNTIL_LAST_FREE_FLAG": "syscall", - "syscall.CERT_STORE_PROV_MEMORY": "syscall", - "syscall.CERT_TRUST_HAS_EXCLUDED_NAME_CONSTRAINT": "syscall", - "syscall.CERT_TRUST_HAS_NOT_DEFINED_NAME_CONSTRAINT": "syscall", - "syscall.CERT_TRUST_HAS_NOT_PERMITTED_NAME_CONSTRAINT": "syscall", - "syscall.CERT_TRUST_HAS_NOT_SUPPORTED_CRITICAL_EXT": "syscall", - "syscall.CERT_TRUST_HAS_NOT_SUPPORTED_NAME_CONSTRAINT": "syscall", - "syscall.CERT_TRUST_INVALID_BASIC_CONSTRAINTS": "syscall", - "syscall.CERT_TRUST_INVALID_EXTENSION": "syscall", - "syscall.CERT_TRUST_INVALID_NAME_CONSTRAINTS": "syscall", - "syscall.CERT_TRUST_INVALID_POLICY_CONSTRAINTS": "syscall", - "syscall.CERT_TRUST_IS_CYCLIC": "syscall", - "syscall.CERT_TRUST_IS_EXPLICIT_DISTRUST": "syscall", - "syscall.CERT_TRUST_IS_NOT_SIGNATURE_VALID": "syscall", - "syscall.CERT_TRUST_IS_NOT_TIME_VALID": "syscall", - "syscall.CERT_TRUST_IS_NOT_VALID_FOR_USAGE": "syscall", - "syscall.CERT_TRUST_IS_OFFLINE_REVOCATION": "syscall", - "syscall.CERT_TRUST_IS_REVOKED": "syscall", - "syscall.CERT_TRUST_IS_UNTRUSTED_ROOT": "syscall", - "syscall.CERT_TRUST_NO_ERROR": "syscall", - "syscall.CERT_TRUST_NO_ISSUANCE_CHAIN_POLICY": "syscall", - "syscall.CERT_TRUST_REVOCATION_STATUS_UNKNOWN": "syscall", - "syscall.CFLUSH": "syscall", - "syscall.CLOCAL": "syscall", - "syscall.CLONE_CHILD_CLEARTID": "syscall", - "syscall.CLONE_CHILD_SETTID": "syscall", - "syscall.CLONE_CSIGNAL": "syscall", - "syscall.CLONE_DETACHED": "syscall", - "syscall.CLONE_FILES": "syscall", - "syscall.CLONE_FS": "syscall", - "syscall.CLONE_IO": "syscall", - "syscall.CLONE_NEWIPC": "syscall", - "syscall.CLONE_NEWNET": "syscall", - "syscall.CLONE_NEWNS": "syscall", - "syscall.CLONE_NEWPID": "syscall", - "syscall.CLONE_NEWUSER": "syscall", - "syscall.CLONE_NEWUTS": "syscall", - "syscall.CLONE_PARENT": "syscall", - "syscall.CLONE_PARENT_SETTID": "syscall", - "syscall.CLONE_PID": "syscall", - "syscall.CLONE_PTRACE": "syscall", - "syscall.CLONE_SETTLS": "syscall", - "syscall.CLONE_SIGHAND": "syscall", - "syscall.CLONE_SYSVSEM": "syscall", - "syscall.CLONE_THREAD": "syscall", - "syscall.CLONE_UNTRACED": "syscall", - "syscall.CLONE_VFORK": "syscall", - "syscall.CLONE_VM": "syscall", - "syscall.CPUID_CFLUSH": "syscall", - "syscall.CREAD": "syscall", - "syscall.CREATE_ALWAYS": "syscall", - "syscall.CREATE_NEW": "syscall", - "syscall.CREATE_NEW_PROCESS_GROUP": "syscall", - "syscall.CREATE_UNICODE_ENVIRONMENT": "syscall", - "syscall.CRYPT_DEFAULT_CONTAINER_OPTIONAL": "syscall", - "syscall.CRYPT_DELETEKEYSET": "syscall", - "syscall.CRYPT_MACHINE_KEYSET": "syscall", - "syscall.CRYPT_NEWKEYSET": "syscall", - "syscall.CRYPT_SILENT": "syscall", - "syscall.CRYPT_VERIFYCONTEXT": "syscall", - "syscall.CS5": "syscall", - "syscall.CS6": "syscall", - "syscall.CS7": "syscall", - "syscall.CS8": "syscall", - "syscall.CSIZE": "syscall", - "syscall.CSTART": "syscall", - "syscall.CSTATUS": "syscall", - "syscall.CSTOP": "syscall", - "syscall.CSTOPB": "syscall", - "syscall.CSUSP": "syscall", - "syscall.CTL_MAXNAME": "syscall", - "syscall.CTL_NET": "syscall", - "syscall.CTL_QUERY": "syscall", - "syscall.CTRL_BREAK_EVENT": "syscall", - "syscall.CTRL_C_EVENT": "syscall", - "syscall.CancelIo": "syscall", - "syscall.CancelIoEx": "syscall", - "syscall.CertAddCertificateContextToStore": "syscall", - "syscall.CertChainContext": "syscall", - "syscall.CertChainElement": "syscall", - "syscall.CertChainPara": "syscall", - "syscall.CertChainPolicyPara": "syscall", - "syscall.CertChainPolicyStatus": "syscall", - "syscall.CertCloseStore": "syscall", - "syscall.CertContext": "syscall", - "syscall.CertCreateCertificateContext": "syscall", - "syscall.CertEnhKeyUsage": "syscall", - "syscall.CertEnumCertificatesInStore": "syscall", - "syscall.CertFreeCertificateChain": "syscall", - "syscall.CertFreeCertificateContext": "syscall", - "syscall.CertGetCertificateChain": "syscall", - "syscall.CertOpenStore": "syscall", - "syscall.CertOpenSystemStore": "syscall", - "syscall.CertRevocationInfo": "syscall", - "syscall.CertSimpleChain": "syscall", - "syscall.CertTrustStatus": "syscall", - "syscall.CertUsageMatch": "syscall", - "syscall.CertVerifyCertificateChainPolicy": "syscall", - "syscall.Chdir": "syscall", - "syscall.CheckBpfVersion": "syscall", - "syscall.Chflags": "syscall", - "syscall.Chmod": "syscall", - "syscall.Chown": "syscall", - "syscall.Chroot": "syscall", - "syscall.Clearenv": "syscall", - "syscall.Close": "syscall", - "syscall.CloseHandle": "syscall", - "syscall.CloseOnExec": "syscall", - "syscall.Closesocket": "syscall", - "syscall.CmsgLen": "syscall", - "syscall.CmsgSpace": "syscall", - "syscall.Cmsghdr": "syscall", - "syscall.CommandLineToArgv": "syscall", - "syscall.ComputerName": "syscall", - "syscall.Connect": "syscall", - "syscall.ConnectEx": "syscall", - "syscall.ConvertSidToStringSid": "syscall", - "syscall.ConvertStringSidToSid": "syscall", - "syscall.CopySid": "syscall", - "syscall.Creat": "syscall", - "syscall.CreateDirectory": "syscall", - "syscall.CreateFile": "syscall", - "syscall.CreateFileMapping": "syscall", - "syscall.CreateHardLink": "syscall", - "syscall.CreateIoCompletionPort": "syscall", - "syscall.CreatePipe": "syscall", - "syscall.CreateProcess": "syscall", - "syscall.CreateSymbolicLink": "syscall", - "syscall.CreateToolhelp32Snapshot": "syscall", - "syscall.Credential": "syscall", - "syscall.CryptAcquireContext": "syscall", - "syscall.CryptGenRandom": "syscall", - "syscall.CryptReleaseContext": "syscall", - "syscall.DIOCBSFLUSH": "syscall", - "syscall.DIOCOSFPFLUSH": "syscall", - "syscall.DLL": "syscall", - "syscall.DLLError": "syscall", - "syscall.DLT_A429": "syscall", - "syscall.DLT_A653_ICM": "syscall", - "syscall.DLT_AIRONET_HEADER": "syscall", - "syscall.DLT_AOS": "syscall", - "syscall.DLT_APPLE_IP_OVER_IEEE1394": "syscall", - "syscall.DLT_ARCNET": "syscall", - "syscall.DLT_ARCNET_LINUX": "syscall", - "syscall.DLT_ATM_CLIP": "syscall", - "syscall.DLT_ATM_RFC1483": "syscall", - "syscall.DLT_AURORA": "syscall", - "syscall.DLT_AX25": "syscall", - "syscall.DLT_AX25_KISS": "syscall", - "syscall.DLT_BACNET_MS_TP": "syscall", - "syscall.DLT_BLUETOOTH_HCI_H4": "syscall", - "syscall.DLT_BLUETOOTH_HCI_H4_WITH_PHDR": "syscall", - "syscall.DLT_CAN20B": "syscall", - "syscall.DLT_CAN_SOCKETCAN": "syscall", - "syscall.DLT_CHAOS": "syscall", - "syscall.DLT_CHDLC": "syscall", - "syscall.DLT_CISCO_IOS": "syscall", - "syscall.DLT_C_HDLC": "syscall", - "syscall.DLT_C_HDLC_WITH_DIR": "syscall", - "syscall.DLT_DBUS": "syscall", - "syscall.DLT_DECT": "syscall", - "syscall.DLT_DOCSIS": "syscall", - "syscall.DLT_DVB_CI": "syscall", - "syscall.DLT_ECONET": "syscall", - "syscall.DLT_EN10MB": "syscall", - "syscall.DLT_EN3MB": "syscall", - "syscall.DLT_ENC": "syscall", - "syscall.DLT_ERF": "syscall", - "syscall.DLT_ERF_ETH": "syscall", - "syscall.DLT_ERF_POS": "syscall", - "syscall.DLT_FC_2": "syscall", - "syscall.DLT_FC_2_WITH_FRAME_DELIMS": "syscall", - "syscall.DLT_FDDI": "syscall", - "syscall.DLT_FLEXRAY": "syscall", - "syscall.DLT_FRELAY": "syscall", - "syscall.DLT_FRELAY_WITH_DIR": "syscall", - "syscall.DLT_GCOM_SERIAL": "syscall", - "syscall.DLT_GCOM_T1E1": "syscall", - "syscall.DLT_GPF_F": "syscall", - "syscall.DLT_GPF_T": "syscall", - "syscall.DLT_GPRS_LLC": "syscall", - "syscall.DLT_GSMTAP_ABIS": "syscall", - "syscall.DLT_GSMTAP_UM": "syscall", - "syscall.DLT_HDLC": "syscall", - "syscall.DLT_HHDLC": "syscall", - "syscall.DLT_HIPPI": "syscall", - "syscall.DLT_IBM_SN": "syscall", - "syscall.DLT_IBM_SP": "syscall", - "syscall.DLT_IEEE802": "syscall", - "syscall.DLT_IEEE802_11": "syscall", - "syscall.DLT_IEEE802_11_RADIO": "syscall", - "syscall.DLT_IEEE802_11_RADIO_AVS": "syscall", - "syscall.DLT_IEEE802_15_4": "syscall", - "syscall.DLT_IEEE802_15_4_LINUX": "syscall", - "syscall.DLT_IEEE802_15_4_NOFCS": "syscall", - "syscall.DLT_IEEE802_15_4_NONASK_PHY": "syscall", - "syscall.DLT_IEEE802_16_MAC_CPS": "syscall", - "syscall.DLT_IEEE802_16_MAC_CPS_RADIO": "syscall", - "syscall.DLT_IPFILTER": "syscall", - "syscall.DLT_IPMB": "syscall", - "syscall.DLT_IPMB_LINUX": "syscall", - "syscall.DLT_IPNET": "syscall", - "syscall.DLT_IPOIB": "syscall", - "syscall.DLT_IPV4": "syscall", - "syscall.DLT_IPV6": "syscall", - "syscall.DLT_IP_OVER_FC": "syscall", - "syscall.DLT_JUNIPER_ATM1": "syscall", - "syscall.DLT_JUNIPER_ATM2": "syscall", - "syscall.DLT_JUNIPER_ATM_CEMIC": "syscall", - "syscall.DLT_JUNIPER_CHDLC": "syscall", - "syscall.DLT_JUNIPER_ES": "syscall", - "syscall.DLT_JUNIPER_ETHER": "syscall", - "syscall.DLT_JUNIPER_FIBRECHANNEL": "syscall", - "syscall.DLT_JUNIPER_FRELAY": "syscall", - "syscall.DLT_JUNIPER_GGSN": "syscall", - "syscall.DLT_JUNIPER_ISM": "syscall", - "syscall.DLT_JUNIPER_MFR": "syscall", - "syscall.DLT_JUNIPER_MLFR": "syscall", - "syscall.DLT_JUNIPER_MLPPP": "syscall", - "syscall.DLT_JUNIPER_MONITOR": "syscall", - "syscall.DLT_JUNIPER_PIC_PEER": "syscall", - "syscall.DLT_JUNIPER_PPP": "syscall", - "syscall.DLT_JUNIPER_PPPOE": "syscall", - "syscall.DLT_JUNIPER_PPPOE_ATM": "syscall", - "syscall.DLT_JUNIPER_SERVICES": "syscall", - "syscall.DLT_JUNIPER_SRX_E2E": "syscall", - "syscall.DLT_JUNIPER_ST": "syscall", - "syscall.DLT_JUNIPER_VP": "syscall", - "syscall.DLT_JUNIPER_VS": "syscall", - "syscall.DLT_LAPB_WITH_DIR": "syscall", - "syscall.DLT_LAPD": "syscall", - "syscall.DLT_LIN": "syscall", - "syscall.DLT_LINUX_EVDEV": "syscall", - "syscall.DLT_LINUX_IRDA": "syscall", - "syscall.DLT_LINUX_LAPD": "syscall", - "syscall.DLT_LINUX_PPP_WITHDIRECTION": "syscall", - "syscall.DLT_LINUX_SLL": "syscall", - "syscall.DLT_LOOP": "syscall", - "syscall.DLT_LTALK": "syscall", - "syscall.DLT_MATCHING_MAX": "syscall", - "syscall.DLT_MATCHING_MIN": "syscall", - "syscall.DLT_MFR": "syscall", - "syscall.DLT_MOST": "syscall", - "syscall.DLT_MPEG_2_TS": "syscall", - "syscall.DLT_MPLS": "syscall", - "syscall.DLT_MTP2": "syscall", - "syscall.DLT_MTP2_WITH_PHDR": "syscall", - "syscall.DLT_MTP3": "syscall", - "syscall.DLT_MUX27010": "syscall", - "syscall.DLT_NETANALYZER": "syscall", - "syscall.DLT_NETANALYZER_TRANSPARENT": "syscall", - "syscall.DLT_NFC_LLCP": "syscall", - "syscall.DLT_NFLOG": "syscall", - "syscall.DLT_NG40": "syscall", - "syscall.DLT_NULL": "syscall", - "syscall.DLT_PCI_EXP": "syscall", - "syscall.DLT_PFLOG": "syscall", - "syscall.DLT_PFSYNC": "syscall", - "syscall.DLT_PPI": "syscall", - "syscall.DLT_PPP": "syscall", - "syscall.DLT_PPP_BSDOS": "syscall", - "syscall.DLT_PPP_ETHER": "syscall", - "syscall.DLT_PPP_PPPD": "syscall", - "syscall.DLT_PPP_SERIAL": "syscall", - "syscall.DLT_PPP_WITH_DIR": "syscall", - "syscall.DLT_PPP_WITH_DIRECTION": "syscall", - "syscall.DLT_PRISM_HEADER": "syscall", - "syscall.DLT_PRONET": "syscall", - "syscall.DLT_RAIF1": "syscall", - "syscall.DLT_RAW": "syscall", - "syscall.DLT_RAWAF_MASK": "syscall", - "syscall.DLT_RIO": "syscall", - "syscall.DLT_SCCP": "syscall", - "syscall.DLT_SITA": "syscall", - "syscall.DLT_SLIP": "syscall", - "syscall.DLT_SLIP_BSDOS": "syscall", - "syscall.DLT_STANAG_5066_D_PDU": "syscall", - "syscall.DLT_SUNATM": "syscall", - "syscall.DLT_SYMANTEC_FIREWALL": "syscall", - "syscall.DLT_TZSP": "syscall", - "syscall.DLT_USB": "syscall", - "syscall.DLT_USB_LINUX": "syscall", - "syscall.DLT_USB_LINUX_MMAPPED": "syscall", - "syscall.DLT_USER0": "syscall", - "syscall.DLT_USER1": "syscall", - "syscall.DLT_USER10": "syscall", - "syscall.DLT_USER11": "syscall", - "syscall.DLT_USER12": "syscall", - "syscall.DLT_USER13": "syscall", - "syscall.DLT_USER14": "syscall", - "syscall.DLT_USER15": "syscall", - "syscall.DLT_USER2": "syscall", - "syscall.DLT_USER3": "syscall", - "syscall.DLT_USER4": "syscall", - "syscall.DLT_USER5": "syscall", - "syscall.DLT_USER6": "syscall", - "syscall.DLT_USER7": "syscall", - "syscall.DLT_USER8": "syscall", - "syscall.DLT_USER9": "syscall", - "syscall.DLT_WIHART": "syscall", - "syscall.DLT_X2E_SERIAL": "syscall", - "syscall.DLT_X2E_XORAYA": "syscall", - "syscall.DNSMXData": "syscall", - "syscall.DNSPTRData": "syscall", - "syscall.DNSRecord": "syscall", - "syscall.DNSSRVData": "syscall", - "syscall.DNSTXTData": "syscall", - "syscall.DNS_INFO_NO_RECORDS": "syscall", - "syscall.DNS_TYPE_A": "syscall", - "syscall.DNS_TYPE_A6": "syscall", - "syscall.DNS_TYPE_AAAA": "syscall", - "syscall.DNS_TYPE_ADDRS": "syscall", - "syscall.DNS_TYPE_AFSDB": "syscall", - "syscall.DNS_TYPE_ALL": "syscall", - "syscall.DNS_TYPE_ANY": "syscall", - "syscall.DNS_TYPE_ATMA": "syscall", - "syscall.DNS_TYPE_AXFR": "syscall", - "syscall.DNS_TYPE_CERT": "syscall", - "syscall.DNS_TYPE_CNAME": "syscall", - "syscall.DNS_TYPE_DHCID": "syscall", - "syscall.DNS_TYPE_DNAME": "syscall", - "syscall.DNS_TYPE_DNSKEY": "syscall", - "syscall.DNS_TYPE_DS": "syscall", - "syscall.DNS_TYPE_EID": "syscall", - "syscall.DNS_TYPE_GID": "syscall", - "syscall.DNS_TYPE_GPOS": "syscall", - "syscall.DNS_TYPE_HINFO": "syscall", - "syscall.DNS_TYPE_ISDN": "syscall", - "syscall.DNS_TYPE_IXFR": "syscall", - "syscall.DNS_TYPE_KEY": "syscall", - "syscall.DNS_TYPE_KX": "syscall", - "syscall.DNS_TYPE_LOC": "syscall", - "syscall.DNS_TYPE_MAILA": "syscall", - "syscall.DNS_TYPE_MAILB": "syscall", - "syscall.DNS_TYPE_MB": "syscall", - "syscall.DNS_TYPE_MD": "syscall", - "syscall.DNS_TYPE_MF": "syscall", - "syscall.DNS_TYPE_MG": "syscall", - "syscall.DNS_TYPE_MINFO": "syscall", - "syscall.DNS_TYPE_MR": "syscall", - "syscall.DNS_TYPE_MX": "syscall", - "syscall.DNS_TYPE_NAPTR": "syscall", - "syscall.DNS_TYPE_NBSTAT": "syscall", - "syscall.DNS_TYPE_NIMLOC": "syscall", - "syscall.DNS_TYPE_NS": "syscall", - "syscall.DNS_TYPE_NSAP": "syscall", - "syscall.DNS_TYPE_NSAPPTR": "syscall", - "syscall.DNS_TYPE_NSEC": "syscall", - "syscall.DNS_TYPE_NULL": "syscall", - "syscall.DNS_TYPE_NXT": "syscall", - "syscall.DNS_TYPE_OPT": "syscall", - "syscall.DNS_TYPE_PTR": "syscall", - "syscall.DNS_TYPE_PX": "syscall", - "syscall.DNS_TYPE_RP": "syscall", - "syscall.DNS_TYPE_RRSIG": "syscall", - "syscall.DNS_TYPE_RT": "syscall", - "syscall.DNS_TYPE_SIG": "syscall", - "syscall.DNS_TYPE_SINK": "syscall", - "syscall.DNS_TYPE_SOA": "syscall", - "syscall.DNS_TYPE_SRV": "syscall", - "syscall.DNS_TYPE_TEXT": "syscall", - "syscall.DNS_TYPE_TKEY": "syscall", - "syscall.DNS_TYPE_TSIG": "syscall", - "syscall.DNS_TYPE_UID": "syscall", - "syscall.DNS_TYPE_UINFO": "syscall", - "syscall.DNS_TYPE_UNSPEC": "syscall", - "syscall.DNS_TYPE_WINS": "syscall", - "syscall.DNS_TYPE_WINSR": "syscall", - "syscall.DNS_TYPE_WKS": "syscall", - "syscall.DNS_TYPE_X25": "syscall", - "syscall.DT_BLK": "syscall", - "syscall.DT_CHR": "syscall", - "syscall.DT_DIR": "syscall", - "syscall.DT_FIFO": "syscall", - "syscall.DT_LNK": "syscall", - "syscall.DT_REG": "syscall", - "syscall.DT_SOCK": "syscall", - "syscall.DT_UNKNOWN": "syscall", - "syscall.DT_WHT": "syscall", - "syscall.DUPLICATE_CLOSE_SOURCE": "syscall", - "syscall.DUPLICATE_SAME_ACCESS": "syscall", - "syscall.DeleteFile": "syscall", - "syscall.DetachLsf": "syscall", - "syscall.DeviceIoControl": "syscall", - "syscall.Dirent": "syscall", - "syscall.DnsNameCompare": "syscall", - "syscall.DnsQuery": "syscall", - "syscall.DnsRecordListFree": "syscall", - "syscall.DnsSectionAdditional": "syscall", - "syscall.DnsSectionAnswer": "syscall", - "syscall.DnsSectionAuthority": "syscall", - "syscall.DnsSectionQuestion": "syscall", - "syscall.Dup": "syscall", - "syscall.Dup2": "syscall", - "syscall.Dup3": "syscall", - "syscall.DuplicateHandle": "syscall", - "syscall.E2BIG": "syscall", - "syscall.EACCES": "syscall", - "syscall.EADDRINUSE": "syscall", - "syscall.EADDRNOTAVAIL": "syscall", - "syscall.EADV": "syscall", - "syscall.EAFNOSUPPORT": "syscall", - "syscall.EAGAIN": "syscall", - "syscall.EALREADY": "syscall", - "syscall.EAUTH": "syscall", - "syscall.EBADARCH": "syscall", - "syscall.EBADE": "syscall", - "syscall.EBADEXEC": "syscall", - "syscall.EBADF": "syscall", - "syscall.EBADFD": "syscall", - "syscall.EBADMACHO": "syscall", - "syscall.EBADMSG": "syscall", - "syscall.EBADR": "syscall", - "syscall.EBADRPC": "syscall", - "syscall.EBADRQC": "syscall", - "syscall.EBADSLT": "syscall", - "syscall.EBFONT": "syscall", - "syscall.EBUSY": "syscall", - "syscall.ECANCELED": "syscall", - "syscall.ECAPMODE": "syscall", - "syscall.ECHILD": "syscall", - "syscall.ECHO": "syscall", - "syscall.ECHOCTL": "syscall", - "syscall.ECHOE": "syscall", - "syscall.ECHOK": "syscall", - "syscall.ECHOKE": "syscall", - "syscall.ECHONL": "syscall", - "syscall.ECHOPRT": "syscall", - "syscall.ECHRNG": "syscall", - "syscall.ECOMM": "syscall", - "syscall.ECONNABORTED": "syscall", - "syscall.ECONNREFUSED": "syscall", - "syscall.ECONNRESET": "syscall", - "syscall.EDEADLK": "syscall", - "syscall.EDEADLOCK": "syscall", - "syscall.EDESTADDRREQ": "syscall", - "syscall.EDEVERR": "syscall", - "syscall.EDOM": "syscall", - "syscall.EDOOFUS": "syscall", - "syscall.EDOTDOT": "syscall", - "syscall.EDQUOT": "syscall", - "syscall.EEXIST": "syscall", - "syscall.EFAULT": "syscall", - "syscall.EFBIG": "syscall", - "syscall.EFER_LMA": "syscall", - "syscall.EFER_LME": "syscall", - "syscall.EFER_NXE": "syscall", - "syscall.EFER_SCE": "syscall", - "syscall.EFTYPE": "syscall", - "syscall.EHOSTDOWN": "syscall", - "syscall.EHOSTUNREACH": "syscall", - "syscall.EHWPOISON": "syscall", - "syscall.EIDRM": "syscall", - "syscall.EILSEQ": "syscall", - "syscall.EINPROGRESS": "syscall", - "syscall.EINTR": "syscall", - "syscall.EINVAL": "syscall", - "syscall.EIO": "syscall", - "syscall.EIPSEC": "syscall", - "syscall.EISCONN": "syscall", - "syscall.EISDIR": "syscall", - "syscall.EISNAM": "syscall", - "syscall.EKEYEXPIRED": "syscall", - "syscall.EKEYREJECTED": "syscall", - "syscall.EKEYREVOKED": "syscall", - "syscall.EL2HLT": "syscall", - "syscall.EL2NSYNC": "syscall", - "syscall.EL3HLT": "syscall", - "syscall.EL3RST": "syscall", - "syscall.ELAST": "syscall", - "syscall.ELF_NGREG": "syscall", - "syscall.ELF_PRARGSZ": "syscall", - "syscall.ELIBACC": "syscall", - "syscall.ELIBBAD": "syscall", - "syscall.ELIBEXEC": "syscall", - "syscall.ELIBMAX": "syscall", - "syscall.ELIBSCN": "syscall", - "syscall.ELNRNG": "syscall", - "syscall.ELOOP": "syscall", - "syscall.EMEDIUMTYPE": "syscall", - "syscall.EMFILE": "syscall", - "syscall.EMLINK": "syscall", - "syscall.EMSGSIZE": "syscall", - "syscall.EMT_TAGOVF": "syscall", - "syscall.EMULTIHOP": "syscall", - "syscall.EMUL_ENABLED": "syscall", - "syscall.EMUL_LINUX": "syscall", - "syscall.EMUL_LINUX32": "syscall", - "syscall.EMUL_MAXID": "syscall", - "syscall.EMUL_NATIVE": "syscall", - "syscall.ENAMETOOLONG": "syscall", - "syscall.ENAVAIL": "syscall", - "syscall.ENDRUNDISC": "syscall", - "syscall.ENEEDAUTH": "syscall", - "syscall.ENETDOWN": "syscall", - "syscall.ENETRESET": "syscall", - "syscall.ENETUNREACH": "syscall", - "syscall.ENFILE": "syscall", - "syscall.ENOANO": "syscall", - "syscall.ENOATTR": "syscall", - "syscall.ENOBUFS": "syscall", - "syscall.ENOCSI": "syscall", - "syscall.ENODATA": "syscall", - "syscall.ENODEV": "syscall", - "syscall.ENOENT": "syscall", - "syscall.ENOEXEC": "syscall", - "syscall.ENOKEY": "syscall", - "syscall.ENOLCK": "syscall", - "syscall.ENOLINK": "syscall", - "syscall.ENOMEDIUM": "syscall", - "syscall.ENOMEM": "syscall", - "syscall.ENOMSG": "syscall", - "syscall.ENONET": "syscall", - "syscall.ENOPKG": "syscall", - "syscall.ENOPOLICY": "syscall", - "syscall.ENOPROTOOPT": "syscall", - "syscall.ENOSPC": "syscall", - "syscall.ENOSR": "syscall", - "syscall.ENOSTR": "syscall", - "syscall.ENOSYS": "syscall", - "syscall.ENOTBLK": "syscall", - "syscall.ENOTCAPABLE": "syscall", - "syscall.ENOTCONN": "syscall", - "syscall.ENOTDIR": "syscall", - "syscall.ENOTEMPTY": "syscall", - "syscall.ENOTNAM": "syscall", - "syscall.ENOTRECOVERABLE": "syscall", - "syscall.ENOTSOCK": "syscall", - "syscall.ENOTSUP": "syscall", - "syscall.ENOTTY": "syscall", - "syscall.ENOTUNIQ": "syscall", - "syscall.ENXIO": "syscall", - "syscall.EN_SW_CTL_INF": "syscall", - "syscall.EN_SW_CTL_PREC": "syscall", - "syscall.EN_SW_CTL_ROUND": "syscall", - "syscall.EN_SW_DATACHAIN": "syscall", - "syscall.EN_SW_DENORM": "syscall", - "syscall.EN_SW_INVOP": "syscall", - "syscall.EN_SW_OVERFLOW": "syscall", - "syscall.EN_SW_PRECLOSS": "syscall", - "syscall.EN_SW_UNDERFLOW": "syscall", - "syscall.EN_SW_ZERODIV": "syscall", - "syscall.EOPNOTSUPP": "syscall", - "syscall.EOVERFLOW": "syscall", - "syscall.EOWNERDEAD": "syscall", - "syscall.EPERM": "syscall", - "syscall.EPFNOSUPPORT": "syscall", - "syscall.EPIPE": "syscall", - "syscall.EPOLLERR": "syscall", - "syscall.EPOLLET": "syscall", - "syscall.EPOLLHUP": "syscall", - "syscall.EPOLLIN": "syscall", - "syscall.EPOLLMSG": "syscall", - "syscall.EPOLLONESHOT": "syscall", - "syscall.EPOLLOUT": "syscall", - "syscall.EPOLLPRI": "syscall", - "syscall.EPOLLRDBAND": "syscall", - "syscall.EPOLLRDHUP": "syscall", - "syscall.EPOLLRDNORM": "syscall", - "syscall.EPOLLWRBAND": "syscall", - "syscall.EPOLLWRNORM": "syscall", - "syscall.EPOLL_CLOEXEC": "syscall", - "syscall.EPOLL_CTL_ADD": "syscall", - "syscall.EPOLL_CTL_DEL": "syscall", - "syscall.EPOLL_CTL_MOD": "syscall", - "syscall.EPOLL_NONBLOCK": "syscall", - "syscall.EPROCLIM": "syscall", - "syscall.EPROCUNAVAIL": "syscall", - "syscall.EPROGMISMATCH": "syscall", - "syscall.EPROGUNAVAIL": "syscall", - "syscall.EPROTO": "syscall", - "syscall.EPROTONOSUPPORT": "syscall", - "syscall.EPROTOTYPE": "syscall", - "syscall.EPWROFF": "syscall", - "syscall.ERANGE": "syscall", - "syscall.EREMCHG": "syscall", - "syscall.EREMOTE": "syscall", - "syscall.EREMOTEIO": "syscall", - "syscall.ERESTART": "syscall", - "syscall.ERFKILL": "syscall", - "syscall.EROFS": "syscall", - "syscall.ERPCMISMATCH": "syscall", - "syscall.ERROR_ACCESS_DENIED": "syscall", - "syscall.ERROR_ALREADY_EXISTS": "syscall", - "syscall.ERROR_BROKEN_PIPE": "syscall", - "syscall.ERROR_BUFFER_OVERFLOW": "syscall", - "syscall.ERROR_ENVVAR_NOT_FOUND": "syscall", - "syscall.ERROR_FILE_EXISTS": "syscall", - "syscall.ERROR_FILE_NOT_FOUND": "syscall", - "syscall.ERROR_HANDLE_EOF": "syscall", - "syscall.ERROR_INSUFFICIENT_BUFFER": "syscall", - "syscall.ERROR_IO_PENDING": "syscall", - "syscall.ERROR_MOD_NOT_FOUND": "syscall", - "syscall.ERROR_MORE_DATA": "syscall", - "syscall.ERROR_NETNAME_DELETED": "syscall", - "syscall.ERROR_NOT_FOUND": "syscall", - "syscall.ERROR_NO_MORE_FILES": "syscall", - "syscall.ERROR_OPERATION_ABORTED": "syscall", - "syscall.ERROR_PATH_NOT_FOUND": "syscall", - "syscall.ERROR_PRIVILEGE_NOT_HELD": "syscall", - "syscall.ERROR_PROC_NOT_FOUND": "syscall", - "syscall.ESHLIBVERS": "syscall", - "syscall.ESHUTDOWN": "syscall", - "syscall.ESOCKTNOSUPPORT": "syscall", - "syscall.ESPIPE": "syscall", - "syscall.ESRCH": "syscall", - "syscall.ESRMNT": "syscall", - "syscall.ESTALE": "syscall", - "syscall.ESTRPIPE": "syscall", - "syscall.ETHERCAP_JUMBO_MTU": "syscall", - "syscall.ETHERCAP_VLAN_HWTAGGING": "syscall", - "syscall.ETHERCAP_VLAN_MTU": "syscall", - "syscall.ETHERMIN": "syscall", - "syscall.ETHERMTU": "syscall", - "syscall.ETHERMTU_JUMBO": "syscall", - "syscall.ETHERTYPE_8023": "syscall", - "syscall.ETHERTYPE_AARP": "syscall", - "syscall.ETHERTYPE_ACCTON": "syscall", - "syscall.ETHERTYPE_AEONIC": "syscall", - "syscall.ETHERTYPE_ALPHA": "syscall", - "syscall.ETHERTYPE_AMBER": "syscall", - "syscall.ETHERTYPE_AMOEBA": "syscall", - "syscall.ETHERTYPE_AOE": "syscall", - "syscall.ETHERTYPE_APOLLO": "syscall", - "syscall.ETHERTYPE_APOLLODOMAIN": "syscall", - "syscall.ETHERTYPE_APPLETALK": "syscall", - "syscall.ETHERTYPE_APPLITEK": "syscall", - "syscall.ETHERTYPE_ARGONAUT": "syscall", - "syscall.ETHERTYPE_ARP": "syscall", - "syscall.ETHERTYPE_AT": "syscall", - "syscall.ETHERTYPE_ATALK": "syscall", - "syscall.ETHERTYPE_ATOMIC": "syscall", - "syscall.ETHERTYPE_ATT": "syscall", - "syscall.ETHERTYPE_ATTSTANFORD": "syscall", - "syscall.ETHERTYPE_AUTOPHON": "syscall", - "syscall.ETHERTYPE_AXIS": "syscall", - "syscall.ETHERTYPE_BCLOOP": "syscall", - "syscall.ETHERTYPE_BOFL": "syscall", - "syscall.ETHERTYPE_CABLETRON": "syscall", - "syscall.ETHERTYPE_CHAOS": "syscall", - "syscall.ETHERTYPE_COMDESIGN": "syscall", - "syscall.ETHERTYPE_COMPUGRAPHIC": "syscall", - "syscall.ETHERTYPE_COUNTERPOINT": "syscall", - "syscall.ETHERTYPE_CRONUS": "syscall", - "syscall.ETHERTYPE_CRONUSVLN": "syscall", - "syscall.ETHERTYPE_DCA": "syscall", - "syscall.ETHERTYPE_DDE": "syscall", - "syscall.ETHERTYPE_DEBNI": "syscall", - "syscall.ETHERTYPE_DECAM": "syscall", - "syscall.ETHERTYPE_DECCUST": "syscall", - "syscall.ETHERTYPE_DECDIAG": "syscall", - "syscall.ETHERTYPE_DECDNS": "syscall", - "syscall.ETHERTYPE_DECDTS": "syscall", - "syscall.ETHERTYPE_DECEXPER": "syscall", - "syscall.ETHERTYPE_DECLAST": "syscall", - "syscall.ETHERTYPE_DECLTM": "syscall", - "syscall.ETHERTYPE_DECMUMPS": "syscall", - "syscall.ETHERTYPE_DECNETBIOS": "syscall", - "syscall.ETHERTYPE_DELTACON": "syscall", - "syscall.ETHERTYPE_DIDDLE": "syscall", - "syscall.ETHERTYPE_DLOG1": "syscall", - "syscall.ETHERTYPE_DLOG2": "syscall", - "syscall.ETHERTYPE_DN": "syscall", - "syscall.ETHERTYPE_DOGFIGHT": "syscall", - "syscall.ETHERTYPE_DSMD": "syscall", - "syscall.ETHERTYPE_ECMA": "syscall", - "syscall.ETHERTYPE_ENCRYPT": "syscall", - "syscall.ETHERTYPE_ES": "syscall", - "syscall.ETHERTYPE_EXCELAN": "syscall", - "syscall.ETHERTYPE_EXPERDATA": "syscall", - "syscall.ETHERTYPE_FLIP": "syscall", - "syscall.ETHERTYPE_FLOWCONTROL": "syscall", - "syscall.ETHERTYPE_FRARP": "syscall", - "syscall.ETHERTYPE_GENDYN": "syscall", - "syscall.ETHERTYPE_HAYES": "syscall", - "syscall.ETHERTYPE_HIPPI_FP": "syscall", - "syscall.ETHERTYPE_HITACHI": "syscall", - "syscall.ETHERTYPE_HP": "syscall", - "syscall.ETHERTYPE_IEEEPUP": "syscall", - "syscall.ETHERTYPE_IEEEPUPAT": "syscall", - "syscall.ETHERTYPE_IMLBL": "syscall", - "syscall.ETHERTYPE_IMLBLDIAG": "syscall", - "syscall.ETHERTYPE_IP": "syscall", - "syscall.ETHERTYPE_IPAS": "syscall", - "syscall.ETHERTYPE_IPV6": "syscall", - "syscall.ETHERTYPE_IPX": "syscall", - "syscall.ETHERTYPE_IPXNEW": "syscall", - "syscall.ETHERTYPE_KALPANA": "syscall", - "syscall.ETHERTYPE_LANBRIDGE": "syscall", - "syscall.ETHERTYPE_LANPROBE": "syscall", - "syscall.ETHERTYPE_LAT": "syscall", - "syscall.ETHERTYPE_LBACK": "syscall", - "syscall.ETHERTYPE_LITTLE": "syscall", - "syscall.ETHERTYPE_LLDP": "syscall", - "syscall.ETHERTYPE_LOGICRAFT": "syscall", - "syscall.ETHERTYPE_LOOPBACK": "syscall", - "syscall.ETHERTYPE_MATRA": "syscall", - "syscall.ETHERTYPE_MAX": "syscall", - "syscall.ETHERTYPE_MERIT": "syscall", - "syscall.ETHERTYPE_MICP": "syscall", - "syscall.ETHERTYPE_MOPDL": "syscall", - "syscall.ETHERTYPE_MOPRC": "syscall", - "syscall.ETHERTYPE_MOTOROLA": "syscall", - "syscall.ETHERTYPE_MPLS": "syscall", - "syscall.ETHERTYPE_MPLS_MCAST": "syscall", - "syscall.ETHERTYPE_MUMPS": "syscall", - "syscall.ETHERTYPE_NBPCC": "syscall", - "syscall.ETHERTYPE_NBPCLAIM": "syscall", - "syscall.ETHERTYPE_NBPCLREQ": "syscall", - "syscall.ETHERTYPE_NBPCLRSP": "syscall", - "syscall.ETHERTYPE_NBPCREQ": "syscall", - "syscall.ETHERTYPE_NBPCRSP": "syscall", - "syscall.ETHERTYPE_NBPDG": "syscall", - "syscall.ETHERTYPE_NBPDGB": "syscall", - "syscall.ETHERTYPE_NBPDLTE": "syscall", - "syscall.ETHERTYPE_NBPRAR": "syscall", - "syscall.ETHERTYPE_NBPRAS": "syscall", - "syscall.ETHERTYPE_NBPRST": "syscall", - "syscall.ETHERTYPE_NBPSCD": "syscall", - "syscall.ETHERTYPE_NBPVCD": "syscall", - "syscall.ETHERTYPE_NBS": "syscall", - "syscall.ETHERTYPE_NCD": "syscall", - "syscall.ETHERTYPE_NESTAR": "syscall", - "syscall.ETHERTYPE_NETBEUI": "syscall", - "syscall.ETHERTYPE_NOVELL": "syscall", - "syscall.ETHERTYPE_NS": "syscall", - "syscall.ETHERTYPE_NSAT": "syscall", - "syscall.ETHERTYPE_NSCOMPAT": "syscall", - "syscall.ETHERTYPE_NTRAILER": "syscall", - "syscall.ETHERTYPE_OS9": "syscall", - "syscall.ETHERTYPE_OS9NET": "syscall", - "syscall.ETHERTYPE_PACER": "syscall", - "syscall.ETHERTYPE_PAE": "syscall", - "syscall.ETHERTYPE_PCS": "syscall", - "syscall.ETHERTYPE_PLANNING": "syscall", - "syscall.ETHERTYPE_PPP": "syscall", - "syscall.ETHERTYPE_PPPOE": "syscall", - "syscall.ETHERTYPE_PPPOEDISC": "syscall", - "syscall.ETHERTYPE_PRIMENTS": "syscall", - "syscall.ETHERTYPE_PUP": "syscall", - "syscall.ETHERTYPE_PUPAT": "syscall", - "syscall.ETHERTYPE_QINQ": "syscall", - "syscall.ETHERTYPE_RACAL": "syscall", - "syscall.ETHERTYPE_RATIONAL": "syscall", - "syscall.ETHERTYPE_RAWFR": "syscall", - "syscall.ETHERTYPE_RCL": "syscall", - "syscall.ETHERTYPE_RDP": "syscall", - "syscall.ETHERTYPE_RETIX": "syscall", - "syscall.ETHERTYPE_REVARP": "syscall", - "syscall.ETHERTYPE_SCA": "syscall", - "syscall.ETHERTYPE_SECTRA": "syscall", - "syscall.ETHERTYPE_SECUREDATA": "syscall", - "syscall.ETHERTYPE_SGITW": "syscall", - "syscall.ETHERTYPE_SG_BOUNCE": "syscall", - "syscall.ETHERTYPE_SG_DIAG": "syscall", - "syscall.ETHERTYPE_SG_NETGAMES": "syscall", - "syscall.ETHERTYPE_SG_RESV": "syscall", - "syscall.ETHERTYPE_SIMNET": "syscall", - "syscall.ETHERTYPE_SLOW": "syscall", - "syscall.ETHERTYPE_SLOWPROTOCOLS": "syscall", - "syscall.ETHERTYPE_SNA": "syscall", - "syscall.ETHERTYPE_SNMP": "syscall", - "syscall.ETHERTYPE_SONIX": "syscall", - "syscall.ETHERTYPE_SPIDER": "syscall", - "syscall.ETHERTYPE_SPRITE": "syscall", - "syscall.ETHERTYPE_STP": "syscall", - "syscall.ETHERTYPE_TALARIS": "syscall", - "syscall.ETHERTYPE_TALARISMC": "syscall", - "syscall.ETHERTYPE_TCPCOMP": "syscall", - "syscall.ETHERTYPE_TCPSM": "syscall", - "syscall.ETHERTYPE_TEC": "syscall", - "syscall.ETHERTYPE_TIGAN": "syscall", - "syscall.ETHERTYPE_TRAIL": "syscall", - "syscall.ETHERTYPE_TRANSETHER": "syscall", - "syscall.ETHERTYPE_TYMSHARE": "syscall", - "syscall.ETHERTYPE_UBBST": "syscall", - "syscall.ETHERTYPE_UBDEBUG": "syscall", - "syscall.ETHERTYPE_UBDIAGLOOP": "syscall", - "syscall.ETHERTYPE_UBDL": "syscall", - "syscall.ETHERTYPE_UBNIU": "syscall", - "syscall.ETHERTYPE_UBNMC": "syscall", - "syscall.ETHERTYPE_VALID": "syscall", - "syscall.ETHERTYPE_VARIAN": "syscall", - "syscall.ETHERTYPE_VAXELN": "syscall", - "syscall.ETHERTYPE_VEECO": "syscall", - "syscall.ETHERTYPE_VEXP": "syscall", - "syscall.ETHERTYPE_VGLAB": "syscall", - "syscall.ETHERTYPE_VINES": "syscall", - "syscall.ETHERTYPE_VINESECHO": "syscall", - "syscall.ETHERTYPE_VINESLOOP": "syscall", - "syscall.ETHERTYPE_VITAL": "syscall", - "syscall.ETHERTYPE_VLAN": "syscall", - "syscall.ETHERTYPE_VLTLMAN": "syscall", - "syscall.ETHERTYPE_VPROD": "syscall", - "syscall.ETHERTYPE_VURESERVED": "syscall", - "syscall.ETHERTYPE_WATERLOO": "syscall", - "syscall.ETHERTYPE_WELLFLEET": "syscall", - "syscall.ETHERTYPE_X25": "syscall", - "syscall.ETHERTYPE_X75": "syscall", - "syscall.ETHERTYPE_XNSSM": "syscall", - "syscall.ETHERTYPE_XTP": "syscall", - "syscall.ETHER_ADDR_LEN": "syscall", - "syscall.ETHER_ALIGN": "syscall", - "syscall.ETHER_CRC_LEN": "syscall", - "syscall.ETHER_CRC_POLY_BE": "syscall", - "syscall.ETHER_CRC_POLY_LE": "syscall", - "syscall.ETHER_HDR_LEN": "syscall", - "syscall.ETHER_MAX_DIX_LEN": "syscall", - "syscall.ETHER_MAX_LEN": "syscall", - "syscall.ETHER_MAX_LEN_JUMBO": "syscall", - "syscall.ETHER_MIN_LEN": "syscall", - "syscall.ETHER_PPPOE_ENCAP_LEN": "syscall", - "syscall.ETHER_TYPE_LEN": "syscall", - "syscall.ETHER_VLAN_ENCAP_LEN": "syscall", - "syscall.ETH_P_1588": "syscall", - "syscall.ETH_P_8021Q": "syscall", - "syscall.ETH_P_802_2": "syscall", - "syscall.ETH_P_802_3": "syscall", - "syscall.ETH_P_AARP": "syscall", - "syscall.ETH_P_ALL": "syscall", - "syscall.ETH_P_AOE": "syscall", - "syscall.ETH_P_ARCNET": "syscall", - "syscall.ETH_P_ARP": "syscall", - "syscall.ETH_P_ATALK": "syscall", - "syscall.ETH_P_ATMFATE": "syscall", - "syscall.ETH_P_ATMMPOA": "syscall", - "syscall.ETH_P_AX25": "syscall", - "syscall.ETH_P_BPQ": "syscall", - "syscall.ETH_P_CAIF": "syscall", - "syscall.ETH_P_CAN": "syscall", - "syscall.ETH_P_CONTROL": "syscall", - "syscall.ETH_P_CUST": "syscall", - "syscall.ETH_P_DDCMP": "syscall", - "syscall.ETH_P_DEC": "syscall", - "syscall.ETH_P_DIAG": "syscall", - "syscall.ETH_P_DNA_DL": "syscall", - "syscall.ETH_P_DNA_RC": "syscall", - "syscall.ETH_P_DNA_RT": "syscall", - "syscall.ETH_P_DSA": "syscall", - "syscall.ETH_P_ECONET": "syscall", - "syscall.ETH_P_EDSA": "syscall", - "syscall.ETH_P_FCOE": "syscall", - "syscall.ETH_P_FIP": "syscall", - "syscall.ETH_P_HDLC": "syscall", - "syscall.ETH_P_IEEE802154": "syscall", - "syscall.ETH_P_IEEEPUP": "syscall", - "syscall.ETH_P_IEEEPUPAT": "syscall", - "syscall.ETH_P_IP": "syscall", - "syscall.ETH_P_IPV6": "syscall", - "syscall.ETH_P_IPX": "syscall", - "syscall.ETH_P_IRDA": "syscall", - "syscall.ETH_P_LAT": "syscall", - "syscall.ETH_P_LINK_CTL": "syscall", - "syscall.ETH_P_LOCALTALK": "syscall", - "syscall.ETH_P_LOOP": "syscall", - "syscall.ETH_P_MOBITEX": "syscall", - "syscall.ETH_P_MPLS_MC": "syscall", - "syscall.ETH_P_MPLS_UC": "syscall", - "syscall.ETH_P_PAE": "syscall", - "syscall.ETH_P_PAUSE": "syscall", - "syscall.ETH_P_PHONET": "syscall", - "syscall.ETH_P_PPPTALK": "syscall", - "syscall.ETH_P_PPP_DISC": "syscall", - "syscall.ETH_P_PPP_MP": "syscall", - "syscall.ETH_P_PPP_SES": "syscall", - "syscall.ETH_P_PUP": "syscall", - "syscall.ETH_P_PUPAT": "syscall", - "syscall.ETH_P_RARP": "syscall", - "syscall.ETH_P_SCA": "syscall", - "syscall.ETH_P_SLOW": "syscall", - "syscall.ETH_P_SNAP": "syscall", - "syscall.ETH_P_TEB": "syscall", - "syscall.ETH_P_TIPC": "syscall", - "syscall.ETH_P_TRAILER": "syscall", - "syscall.ETH_P_TR_802_2": "syscall", - "syscall.ETH_P_WAN_PPP": "syscall", - "syscall.ETH_P_WCCP": "syscall", - "syscall.ETH_P_X25": "syscall", - "syscall.ETIME": "syscall", - "syscall.ETIMEDOUT": "syscall", - "syscall.ETOOMANYREFS": "syscall", - "syscall.ETXTBSY": "syscall", - "syscall.EUCLEAN": "syscall", - "syscall.EUNATCH": "syscall", - "syscall.EUSERS": "syscall", - "syscall.EVFILT_AIO": "syscall", - "syscall.EVFILT_FS": "syscall", - "syscall.EVFILT_LIO": "syscall", - "syscall.EVFILT_MACHPORT": "syscall", - "syscall.EVFILT_PROC": "syscall", - "syscall.EVFILT_READ": "syscall", - "syscall.EVFILT_SIGNAL": "syscall", - "syscall.EVFILT_SYSCOUNT": "syscall", - "syscall.EVFILT_THREADMARKER": "syscall", - "syscall.EVFILT_TIMER": "syscall", - "syscall.EVFILT_USER": "syscall", - "syscall.EVFILT_VM": "syscall", - "syscall.EVFILT_VNODE": "syscall", - "syscall.EVFILT_WRITE": "syscall", - "syscall.EV_ADD": "syscall", - "syscall.EV_CLEAR": "syscall", - "syscall.EV_DELETE": "syscall", - "syscall.EV_DISABLE": "syscall", - "syscall.EV_DISPATCH": "syscall", - "syscall.EV_DROP": "syscall", - "syscall.EV_ENABLE": "syscall", - "syscall.EV_EOF": "syscall", - "syscall.EV_ERROR": "syscall", - "syscall.EV_FLAG0": "syscall", - "syscall.EV_FLAG1": "syscall", - "syscall.EV_ONESHOT": "syscall", - "syscall.EV_OOBAND": "syscall", - "syscall.EV_POLL": "syscall", - "syscall.EV_RECEIPT": "syscall", - "syscall.EV_SYSFLAGS": "syscall", - "syscall.EWINDOWS": "syscall", - "syscall.EWOULDBLOCK": "syscall", - "syscall.EXDEV": "syscall", - "syscall.EXFULL": "syscall", - "syscall.EXTA": "syscall", - "syscall.EXTB": "syscall", - "syscall.EXTPROC": "syscall", - "syscall.Environ": "syscall", - "syscall.EpollCreate": "syscall", - "syscall.EpollCreate1": "syscall", - "syscall.EpollCtl": "syscall", - "syscall.EpollEvent": "syscall", - "syscall.EpollWait": "syscall", - "syscall.Errno": "syscall", - "syscall.EscapeArg": "syscall", - "syscall.Exchangedata": "syscall", - "syscall.Exec": "syscall", - "syscall.Exit": "syscall", - "syscall.ExitProcess": "syscall", - "syscall.FD_CLOEXEC": "syscall", - "syscall.FD_SETSIZE": "syscall", - "syscall.FILE_ACTION_ADDED": "syscall", - "syscall.FILE_ACTION_MODIFIED": "syscall", - "syscall.FILE_ACTION_REMOVED": "syscall", - "syscall.FILE_ACTION_RENAMED_NEW_NAME": "syscall", - "syscall.FILE_ACTION_RENAMED_OLD_NAME": "syscall", - "syscall.FILE_APPEND_DATA": "syscall", - "syscall.FILE_ATTRIBUTE_ARCHIVE": "syscall", - "syscall.FILE_ATTRIBUTE_DIRECTORY": "syscall", - "syscall.FILE_ATTRIBUTE_HIDDEN": "syscall", - "syscall.FILE_ATTRIBUTE_NORMAL": "syscall", - "syscall.FILE_ATTRIBUTE_READONLY": "syscall", - "syscall.FILE_ATTRIBUTE_REPARSE_POINT": "syscall", - "syscall.FILE_ATTRIBUTE_SYSTEM": "syscall", - "syscall.FILE_BEGIN": "syscall", - "syscall.FILE_CURRENT": "syscall", - "syscall.FILE_END": "syscall", - "syscall.FILE_FLAG_BACKUP_SEMANTICS": "syscall", - "syscall.FILE_FLAG_OPEN_REPARSE_POINT": "syscall", - "syscall.FILE_FLAG_OVERLAPPED": "syscall", - "syscall.FILE_LIST_DIRECTORY": "syscall", - "syscall.FILE_MAP_COPY": "syscall", - "syscall.FILE_MAP_EXECUTE": "syscall", - "syscall.FILE_MAP_READ": "syscall", - "syscall.FILE_MAP_WRITE": "syscall", - "syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES": "syscall", - "syscall.FILE_NOTIFY_CHANGE_CREATION": "syscall", - "syscall.FILE_NOTIFY_CHANGE_DIR_NAME": "syscall", - "syscall.FILE_NOTIFY_CHANGE_FILE_NAME": "syscall", - "syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS": "syscall", - "syscall.FILE_NOTIFY_CHANGE_LAST_WRITE": "syscall", - "syscall.FILE_NOTIFY_CHANGE_SIZE": "syscall", - "syscall.FILE_SHARE_DELETE": "syscall", - "syscall.FILE_SHARE_READ": "syscall", - "syscall.FILE_SHARE_WRITE": "syscall", - "syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS": "syscall", - "syscall.FILE_SKIP_SET_EVENT_ON_HANDLE": "syscall", - "syscall.FILE_TYPE_CHAR": "syscall", - "syscall.FILE_TYPE_DISK": "syscall", - "syscall.FILE_TYPE_PIPE": "syscall", - "syscall.FILE_TYPE_REMOTE": "syscall", - "syscall.FILE_TYPE_UNKNOWN": "syscall", - "syscall.FILE_WRITE_ATTRIBUTES": "syscall", - "syscall.FLUSHO": "syscall", - "syscall.FORMAT_MESSAGE_ALLOCATE_BUFFER": "syscall", - "syscall.FORMAT_MESSAGE_ARGUMENT_ARRAY": "syscall", - "syscall.FORMAT_MESSAGE_FROM_HMODULE": "syscall", - "syscall.FORMAT_MESSAGE_FROM_STRING": "syscall", - "syscall.FORMAT_MESSAGE_FROM_SYSTEM": "syscall", - "syscall.FORMAT_MESSAGE_IGNORE_INSERTS": "syscall", - "syscall.FORMAT_MESSAGE_MAX_WIDTH_MASK": "syscall", - "syscall.FSCTL_GET_REPARSE_POINT": "syscall", - "syscall.F_ADDFILESIGS": "syscall", - "syscall.F_ADDSIGS": "syscall", - "syscall.F_ALLOCATEALL": "syscall", - "syscall.F_ALLOCATECONTIG": "syscall", - "syscall.F_CANCEL": "syscall", - "syscall.F_CHKCLEAN": "syscall", - "syscall.F_CLOSEM": "syscall", - "syscall.F_DUP2FD": "syscall", - "syscall.F_DUP2FD_CLOEXEC": "syscall", - "syscall.F_DUPFD": "syscall", - "syscall.F_DUPFD_CLOEXEC": "syscall", - "syscall.F_EXLCK": "syscall", - "syscall.F_FLUSH_DATA": "syscall", - "syscall.F_FREEZE_FS": "syscall", - "syscall.F_FSCTL": "syscall", - "syscall.F_FSDIRMASK": "syscall", - "syscall.F_FSIN": "syscall", - "syscall.F_FSINOUT": "syscall", - "syscall.F_FSOUT": "syscall", - "syscall.F_FSPRIV": "syscall", - "syscall.F_FSVOID": "syscall", - "syscall.F_FULLFSYNC": "syscall", - "syscall.F_GETFD": "syscall", - "syscall.F_GETFL": "syscall", - "syscall.F_GETLEASE": "syscall", - "syscall.F_GETLK": "syscall", - "syscall.F_GETLK64": "syscall", - "syscall.F_GETLKPID": "syscall", - "syscall.F_GETNOSIGPIPE": "syscall", - "syscall.F_GETOWN": "syscall", - "syscall.F_GETOWN_EX": "syscall", - "syscall.F_GETPATH": "syscall", - "syscall.F_GETPATH_MTMINFO": "syscall", - "syscall.F_GETPIPE_SZ": "syscall", - "syscall.F_GETPROTECTIONCLASS": "syscall", - "syscall.F_GETSIG": "syscall", - "syscall.F_GLOBAL_NOCACHE": "syscall", - "syscall.F_LOCK": "syscall", - "syscall.F_LOG2PHYS": "syscall", - "syscall.F_LOG2PHYS_EXT": "syscall", - "syscall.F_MARKDEPENDENCY": "syscall", - "syscall.F_MAXFD": "syscall", - "syscall.F_NOCACHE": "syscall", - "syscall.F_NODIRECT": "syscall", - "syscall.F_NOTIFY": "syscall", - "syscall.F_OGETLK": "syscall", - "syscall.F_OK": "syscall", - "syscall.F_OSETLK": "syscall", - "syscall.F_OSETLKW": "syscall", - "syscall.F_PARAM_MASK": "syscall", - "syscall.F_PARAM_MAX": "syscall", - "syscall.F_PATHPKG_CHECK": "syscall", - "syscall.F_PEOFPOSMODE": "syscall", - "syscall.F_PREALLOCATE": "syscall", - "syscall.F_RDADVISE": "syscall", - "syscall.F_RDAHEAD": "syscall", - "syscall.F_RDLCK": "syscall", - "syscall.F_READAHEAD": "syscall", - "syscall.F_READBOOTSTRAP": "syscall", - "syscall.F_SETBACKINGSTORE": "syscall", - "syscall.F_SETFD": "syscall", - "syscall.F_SETFL": "syscall", - "syscall.F_SETLEASE": "syscall", - "syscall.F_SETLK": "syscall", - "syscall.F_SETLK64": "syscall", - "syscall.F_SETLKW": "syscall", - "syscall.F_SETLKW64": "syscall", - "syscall.F_SETLK_REMOTE": "syscall", - "syscall.F_SETNOSIGPIPE": "syscall", - "syscall.F_SETOWN": "syscall", - "syscall.F_SETOWN_EX": "syscall", - "syscall.F_SETPIPE_SZ": "syscall", - "syscall.F_SETPROTECTIONCLASS": "syscall", - "syscall.F_SETSIG": "syscall", - "syscall.F_SETSIZE": "syscall", - "syscall.F_SHLCK": "syscall", - "syscall.F_TEST": "syscall", - "syscall.F_THAW_FS": "syscall", - "syscall.F_TLOCK": "syscall", - "syscall.F_ULOCK": "syscall", - "syscall.F_UNLCK": "syscall", - "syscall.F_UNLCKSYS": "syscall", - "syscall.F_VOLPOSMODE": "syscall", - "syscall.F_WRITEBOOTSTRAP": "syscall", - "syscall.F_WRLCK": "syscall", - "syscall.Faccessat": "syscall", - "syscall.Fallocate": "syscall", - "syscall.Fbootstraptransfer_t": "syscall", - "syscall.Fchdir": "syscall", - "syscall.Fchflags": "syscall", - "syscall.Fchmod": "syscall", - "syscall.Fchmodat": "syscall", - "syscall.Fchown": "syscall", - "syscall.Fchownat": "syscall", - "syscall.FcntlFlock": "syscall", - "syscall.FdSet": "syscall", - "syscall.Fdatasync": "syscall", - "syscall.FileNotifyInformation": "syscall", - "syscall.Filetime": "syscall", - "syscall.FindClose": "syscall", - "syscall.FindFirstFile": "syscall", - "syscall.FindNextFile": "syscall", - "syscall.Flock": "syscall", - "syscall.Flock_t": "syscall", - "syscall.FlushBpf": "syscall", - "syscall.FlushFileBuffers": "syscall", - "syscall.FlushViewOfFile": "syscall", - "syscall.ForkExec": "syscall", - "syscall.ForkLock": "syscall", - "syscall.FormatMessage": "syscall", - "syscall.Fpathconf": "syscall", - "syscall.FreeAddrInfoW": "syscall", - "syscall.FreeEnvironmentStrings": "syscall", - "syscall.FreeLibrary": "syscall", - "syscall.Fsid": "syscall", - "syscall.Fstat": "syscall", - "syscall.Fstatfs": "syscall", - "syscall.Fstore_t": "syscall", - "syscall.Fsync": "syscall", - "syscall.Ftruncate": "syscall", - "syscall.FullPath": "syscall", - "syscall.Futimes": "syscall", - "syscall.Futimesat": "syscall", - "syscall.GENERIC_ALL": "syscall", - "syscall.GENERIC_EXECUTE": "syscall", - "syscall.GENERIC_READ": "syscall", - "syscall.GENERIC_WRITE": "syscall", - "syscall.GUID": "syscall", - "syscall.GetAcceptExSockaddrs": "syscall", - "syscall.GetAdaptersInfo": "syscall", - "syscall.GetAddrInfoW": "syscall", - "syscall.GetCommandLine": "syscall", - "syscall.GetComputerName": "syscall", - "syscall.GetConsoleMode": "syscall", - "syscall.GetCurrentDirectory": "syscall", - "syscall.GetCurrentProcess": "syscall", - "syscall.GetEnvironmentStrings": "syscall", - "syscall.GetEnvironmentVariable": "syscall", - "syscall.GetExitCodeProcess": "syscall", - "syscall.GetFileAttributes": "syscall", - "syscall.GetFileAttributesEx": "syscall", - "syscall.GetFileExInfoStandard": "syscall", - "syscall.GetFileExMaxInfoLevel": "syscall", - "syscall.GetFileInformationByHandle": "syscall", - "syscall.GetFileType": "syscall", - "syscall.GetFullPathName": "syscall", - "syscall.GetHostByName": "syscall", - "syscall.GetIfEntry": "syscall", - "syscall.GetLastError": "syscall", - "syscall.GetLengthSid": "syscall", - "syscall.GetLongPathName": "syscall", - "syscall.GetProcAddress": "syscall", - "syscall.GetProcessTimes": "syscall", - "syscall.GetProtoByName": "syscall", - "syscall.GetQueuedCompletionStatus": "syscall", - "syscall.GetServByName": "syscall", - "syscall.GetShortPathName": "syscall", - "syscall.GetStartupInfo": "syscall", - "syscall.GetStdHandle": "syscall", - "syscall.GetSystemTimeAsFileTime": "syscall", - "syscall.GetTempPath": "syscall", - "syscall.GetTimeZoneInformation": "syscall", - "syscall.GetTokenInformation": "syscall", - "syscall.GetUserNameEx": "syscall", - "syscall.GetUserProfileDirectory": "syscall", - "syscall.GetVersion": "syscall", - "syscall.Getcwd": "syscall", - "syscall.Getdents": "syscall", - "syscall.Getdirentries": "syscall", - "syscall.Getdtablesize": "syscall", - "syscall.Getegid": "syscall", - "syscall.Getenv": "syscall", - "syscall.Geteuid": "syscall", - "syscall.Getfsstat": "syscall", - "syscall.Getgid": "syscall", - "syscall.Getgroups": "syscall", - "syscall.Getpagesize": "syscall", - "syscall.Getpeername": "syscall", - "syscall.Getpgid": "syscall", - "syscall.Getpgrp": "syscall", - "syscall.Getpid": "syscall", - "syscall.Getppid": "syscall", - "syscall.Getpriority": "syscall", - "syscall.Getrlimit": "syscall", - "syscall.Getrusage": "syscall", - "syscall.Getsid": "syscall", - "syscall.Getsockname": "syscall", - "syscall.Getsockopt": "syscall", - "syscall.GetsockoptByte": "syscall", - "syscall.GetsockoptICMPv6Filter": "syscall", - "syscall.GetsockoptIPMreq": "syscall", - "syscall.GetsockoptIPMreqn": "syscall", - "syscall.GetsockoptIPv6MTUInfo": "syscall", - "syscall.GetsockoptIPv6Mreq": "syscall", - "syscall.GetsockoptInet4Addr": "syscall", - "syscall.GetsockoptInt": "syscall", - "syscall.GetsockoptUcred": "syscall", - "syscall.Gettid": "syscall", - "syscall.Gettimeofday": "syscall", - "syscall.Getuid": "syscall", - "syscall.Getwd": "syscall", - "syscall.Getxattr": "syscall", - "syscall.HANDLE_FLAG_INHERIT": "syscall", - "syscall.HKEY_CLASSES_ROOT": "syscall", - "syscall.HKEY_CURRENT_CONFIG": "syscall", - "syscall.HKEY_CURRENT_USER": "syscall", - "syscall.HKEY_DYN_DATA": "syscall", - "syscall.HKEY_LOCAL_MACHINE": "syscall", - "syscall.HKEY_PERFORMANCE_DATA": "syscall", - "syscall.HKEY_USERS": "syscall", - "syscall.HUPCL": "syscall", - "syscall.Handle": "syscall", - "syscall.Hostent": "syscall", - "syscall.ICANON": "syscall", - "syscall.ICMP6_FILTER": "syscall", - "syscall.ICMPV6_FILTER": "syscall", - "syscall.ICMPv6Filter": "syscall", - "syscall.ICRNL": "syscall", - "syscall.IEXTEN": "syscall", - "syscall.IFAN_ARRIVAL": "syscall", - "syscall.IFAN_DEPARTURE": "syscall", - "syscall.IFA_ADDRESS": "syscall", - "syscall.IFA_ANYCAST": "syscall", - "syscall.IFA_BROADCAST": "syscall", - "syscall.IFA_CACHEINFO": "syscall", - "syscall.IFA_F_DADFAILED": "syscall", - "syscall.IFA_F_DEPRECATED": "syscall", - "syscall.IFA_F_HOMEADDRESS": "syscall", - "syscall.IFA_F_NODAD": "syscall", - "syscall.IFA_F_OPTIMISTIC": "syscall", - "syscall.IFA_F_PERMANENT": "syscall", - "syscall.IFA_F_SECONDARY": "syscall", - "syscall.IFA_F_TEMPORARY": "syscall", - "syscall.IFA_F_TENTATIVE": "syscall", - "syscall.IFA_LABEL": "syscall", - "syscall.IFA_LOCAL": "syscall", - "syscall.IFA_MAX": "syscall", - "syscall.IFA_MULTICAST": "syscall", - "syscall.IFA_ROUTE": "syscall", - "syscall.IFA_UNSPEC": "syscall", - "syscall.IFF_ALLMULTI": "syscall", - "syscall.IFF_ALTPHYS": "syscall", - "syscall.IFF_AUTOMEDIA": "syscall", - "syscall.IFF_BROADCAST": "syscall", - "syscall.IFF_CANTCHANGE": "syscall", - "syscall.IFF_CANTCONFIG": "syscall", - "syscall.IFF_DEBUG": "syscall", - "syscall.IFF_DRV_OACTIVE": "syscall", - "syscall.IFF_DRV_RUNNING": "syscall", - "syscall.IFF_DYING": "syscall", - "syscall.IFF_DYNAMIC": "syscall", - "syscall.IFF_LINK0": "syscall", - "syscall.IFF_LINK1": "syscall", - "syscall.IFF_LINK2": "syscall", - "syscall.IFF_LOOPBACK": "syscall", - "syscall.IFF_MASTER": "syscall", - "syscall.IFF_MONITOR": "syscall", - "syscall.IFF_MULTICAST": "syscall", - "syscall.IFF_NOARP": "syscall", - "syscall.IFF_NOTRAILERS": "syscall", - "syscall.IFF_NO_PI": "syscall", - "syscall.IFF_OACTIVE": "syscall", - "syscall.IFF_ONE_QUEUE": "syscall", - "syscall.IFF_POINTOPOINT": "syscall", - "syscall.IFF_POINTTOPOINT": "syscall", - "syscall.IFF_PORTSEL": "syscall", - "syscall.IFF_PPROMISC": "syscall", - "syscall.IFF_PROMISC": "syscall", - "syscall.IFF_RENAMING": "syscall", - "syscall.IFF_RUNNING": "syscall", - "syscall.IFF_SIMPLEX": "syscall", - "syscall.IFF_SLAVE": "syscall", - "syscall.IFF_SMART": "syscall", - "syscall.IFF_STATICARP": "syscall", - "syscall.IFF_TAP": "syscall", - "syscall.IFF_TUN": "syscall", - "syscall.IFF_TUN_EXCL": "syscall", - "syscall.IFF_UP": "syscall", - "syscall.IFF_VNET_HDR": "syscall", - "syscall.IFLA_ADDRESS": "syscall", - "syscall.IFLA_BROADCAST": "syscall", - "syscall.IFLA_COST": "syscall", - "syscall.IFLA_IFALIAS": "syscall", - "syscall.IFLA_IFNAME": "syscall", - "syscall.IFLA_LINK": "syscall", - "syscall.IFLA_LINKINFO": "syscall", - "syscall.IFLA_LINKMODE": "syscall", - "syscall.IFLA_MAP": "syscall", - "syscall.IFLA_MASTER": "syscall", - "syscall.IFLA_MAX": "syscall", - "syscall.IFLA_MTU": "syscall", - "syscall.IFLA_NET_NS_PID": "syscall", - "syscall.IFLA_OPERSTATE": "syscall", - "syscall.IFLA_PRIORITY": "syscall", - "syscall.IFLA_PROTINFO": "syscall", - "syscall.IFLA_QDISC": "syscall", - "syscall.IFLA_STATS": "syscall", - "syscall.IFLA_TXQLEN": "syscall", - "syscall.IFLA_UNSPEC": "syscall", - "syscall.IFLA_WEIGHT": "syscall", - "syscall.IFLA_WIRELESS": "syscall", - "syscall.IFNAMSIZ": "syscall", - "syscall.IFT_1822": "syscall", - "syscall.IFT_A12MPPSWITCH": "syscall", - "syscall.IFT_AAL2": "syscall", - "syscall.IFT_AAL5": "syscall", - "syscall.IFT_ADSL": "syscall", - "syscall.IFT_AFLANE8023": "syscall", - "syscall.IFT_AFLANE8025": "syscall", - "syscall.IFT_ARAP": "syscall", - "syscall.IFT_ARCNET": "syscall", - "syscall.IFT_ARCNETPLUS": "syscall", - "syscall.IFT_ASYNC": "syscall", - "syscall.IFT_ATM": "syscall", - "syscall.IFT_ATMDXI": "syscall", - "syscall.IFT_ATMFUNI": "syscall", - "syscall.IFT_ATMIMA": "syscall", - "syscall.IFT_ATMLOGICAL": "syscall", - "syscall.IFT_ATMRADIO": "syscall", - "syscall.IFT_ATMSUBINTERFACE": "syscall", - "syscall.IFT_ATMVCIENDPT": "syscall", - "syscall.IFT_ATMVIRTUAL": "syscall", - "syscall.IFT_BGPPOLICYACCOUNTING": "syscall", - "syscall.IFT_BLUETOOTH": "syscall", - "syscall.IFT_BRIDGE": "syscall", - "syscall.IFT_BSC": "syscall", - "syscall.IFT_CARP": "syscall", - "syscall.IFT_CCTEMUL": "syscall", - "syscall.IFT_CELLULAR": "syscall", - "syscall.IFT_CEPT": "syscall", - "syscall.IFT_CES": "syscall", - "syscall.IFT_CHANNEL": "syscall", - "syscall.IFT_CNR": "syscall", - "syscall.IFT_COFFEE": "syscall", - "syscall.IFT_COMPOSITELINK": "syscall", - "syscall.IFT_DCN": "syscall", - "syscall.IFT_DIGITALPOWERLINE": "syscall", - "syscall.IFT_DIGITALWRAPPEROVERHEADCHANNEL": "syscall", - "syscall.IFT_DLSW": "syscall", - "syscall.IFT_DOCSCABLEDOWNSTREAM": "syscall", - "syscall.IFT_DOCSCABLEMACLAYER": "syscall", - "syscall.IFT_DOCSCABLEUPSTREAM": "syscall", - "syscall.IFT_DOCSCABLEUPSTREAMCHANNEL": "syscall", - "syscall.IFT_DS0": "syscall", - "syscall.IFT_DS0BUNDLE": "syscall", - "syscall.IFT_DS1FDL": "syscall", - "syscall.IFT_DS3": "syscall", - "syscall.IFT_DTM": "syscall", - "syscall.IFT_DUMMY": "syscall", - "syscall.IFT_DVBASILN": "syscall", - "syscall.IFT_DVBASIOUT": "syscall", - "syscall.IFT_DVBRCCDOWNSTREAM": "syscall", - "syscall.IFT_DVBRCCMACLAYER": "syscall", - "syscall.IFT_DVBRCCUPSTREAM": "syscall", - "syscall.IFT_ECONET": "syscall", - "syscall.IFT_ENC": "syscall", - "syscall.IFT_EON": "syscall", - "syscall.IFT_EPLRS": "syscall", - "syscall.IFT_ESCON": "syscall", - "syscall.IFT_ETHER": "syscall", - "syscall.IFT_FAITH": "syscall", - "syscall.IFT_FAST": "syscall", - "syscall.IFT_FASTETHER": "syscall", - "syscall.IFT_FASTETHERFX": "syscall", - "syscall.IFT_FDDI": "syscall", - "syscall.IFT_FIBRECHANNEL": "syscall", - "syscall.IFT_FRAMERELAYINTERCONNECT": "syscall", - "syscall.IFT_FRAMERELAYMPI": "syscall", - "syscall.IFT_FRDLCIENDPT": "syscall", - "syscall.IFT_FRELAY": "syscall", - "syscall.IFT_FRELAYDCE": "syscall", - "syscall.IFT_FRF16MFRBUNDLE": "syscall", - "syscall.IFT_FRFORWARD": "syscall", - "syscall.IFT_G703AT2MB": "syscall", - "syscall.IFT_G703AT64K": "syscall", - "syscall.IFT_GIF": "syscall", - "syscall.IFT_GIGABITETHERNET": "syscall", - "syscall.IFT_GR303IDT": "syscall", - "syscall.IFT_GR303RDT": "syscall", - "syscall.IFT_H323GATEKEEPER": "syscall", - "syscall.IFT_H323PROXY": "syscall", - "syscall.IFT_HDH1822": "syscall", - "syscall.IFT_HDLC": "syscall", - "syscall.IFT_HDSL2": "syscall", - "syscall.IFT_HIPERLAN2": "syscall", - "syscall.IFT_HIPPI": "syscall", - "syscall.IFT_HIPPIINTERFACE": "syscall", - "syscall.IFT_HOSTPAD": "syscall", - "syscall.IFT_HSSI": "syscall", - "syscall.IFT_HY": "syscall", - "syscall.IFT_IBM370PARCHAN": "syscall", - "syscall.IFT_IDSL": "syscall", - "syscall.IFT_IEEE1394": "syscall", - "syscall.IFT_IEEE80211": "syscall", - "syscall.IFT_IEEE80212": "syscall", - "syscall.IFT_IEEE8023ADLAG": "syscall", - "syscall.IFT_IFGSN": "syscall", - "syscall.IFT_IMT": "syscall", - "syscall.IFT_INFINIBAND": "syscall", - "syscall.IFT_INTERLEAVE": "syscall", - "syscall.IFT_IP": "syscall", - "syscall.IFT_IPFORWARD": "syscall", - "syscall.IFT_IPOVERATM": "syscall", - "syscall.IFT_IPOVERCDLC": "syscall", - "syscall.IFT_IPOVERCLAW": "syscall", - "syscall.IFT_IPSWITCH": "syscall", - "syscall.IFT_IPXIP": "syscall", - "syscall.IFT_ISDN": "syscall", - "syscall.IFT_ISDNBASIC": "syscall", - "syscall.IFT_ISDNPRIMARY": "syscall", - "syscall.IFT_ISDNS": "syscall", - "syscall.IFT_ISDNU": "syscall", - "syscall.IFT_ISO88022LLC": "syscall", - "syscall.IFT_ISO88023": "syscall", - "syscall.IFT_ISO88024": "syscall", - "syscall.IFT_ISO88025": "syscall", - "syscall.IFT_ISO88025CRFPINT": "syscall", - "syscall.IFT_ISO88025DTR": "syscall", - "syscall.IFT_ISO88025FIBER": "syscall", - "syscall.IFT_ISO88026": "syscall", - "syscall.IFT_ISUP": "syscall", - "syscall.IFT_L2VLAN": "syscall", - "syscall.IFT_L3IPVLAN": "syscall", - "syscall.IFT_L3IPXVLAN": "syscall", - "syscall.IFT_LAPB": "syscall", - "syscall.IFT_LAPD": "syscall", - "syscall.IFT_LAPF": "syscall", - "syscall.IFT_LINEGROUP": "syscall", - "syscall.IFT_LOCALTALK": "syscall", - "syscall.IFT_LOOP": "syscall", - "syscall.IFT_MEDIAMAILOVERIP": "syscall", - "syscall.IFT_MFSIGLINK": "syscall", - "syscall.IFT_MIOX25": "syscall", - "syscall.IFT_MODEM": "syscall", - "syscall.IFT_MPC": "syscall", - "syscall.IFT_MPLS": "syscall", - "syscall.IFT_MPLSTUNNEL": "syscall", - "syscall.IFT_MSDSL": "syscall", - "syscall.IFT_MVL": "syscall", - "syscall.IFT_MYRINET": "syscall", - "syscall.IFT_NFAS": "syscall", - "syscall.IFT_NSIP": "syscall", - "syscall.IFT_OPTICALCHANNEL": "syscall", - "syscall.IFT_OPTICALTRANSPORT": "syscall", - "syscall.IFT_OTHER": "syscall", - "syscall.IFT_P10": "syscall", - "syscall.IFT_P80": "syscall", - "syscall.IFT_PARA": "syscall", - "syscall.IFT_PDP": "syscall", - "syscall.IFT_PFLOG": "syscall", - "syscall.IFT_PFLOW": "syscall", - "syscall.IFT_PFSYNC": "syscall", - "syscall.IFT_PLC": "syscall", - "syscall.IFT_PON155": "syscall", - "syscall.IFT_PON622": "syscall", - "syscall.IFT_POS": "syscall", - "syscall.IFT_PPP": "syscall", - "syscall.IFT_PPPMULTILINKBUNDLE": "syscall", - "syscall.IFT_PROPATM": "syscall", - "syscall.IFT_PROPBWAP2MP": "syscall", - "syscall.IFT_PROPCNLS": "syscall", - "syscall.IFT_PROPDOCSWIRELESSDOWNSTREAM": "syscall", - "syscall.IFT_PROPDOCSWIRELESSMACLAYER": "syscall", - "syscall.IFT_PROPDOCSWIRELESSUPSTREAM": "syscall", - "syscall.IFT_PROPMUX": "syscall", - "syscall.IFT_PROPVIRTUAL": "syscall", - "syscall.IFT_PROPWIRELESSP2P": "syscall", - "syscall.IFT_PTPSERIAL": "syscall", - "syscall.IFT_PVC": "syscall", - "syscall.IFT_Q2931": "syscall", - "syscall.IFT_QLLC": "syscall", - "syscall.IFT_RADIOMAC": "syscall", - "syscall.IFT_RADSL": "syscall", - "syscall.IFT_REACHDSL": "syscall", - "syscall.IFT_RFC1483": "syscall", - "syscall.IFT_RS232": "syscall", - "syscall.IFT_RSRB": "syscall", - "syscall.IFT_SDLC": "syscall", - "syscall.IFT_SDSL": "syscall", - "syscall.IFT_SHDSL": "syscall", - "syscall.IFT_SIP": "syscall", - "syscall.IFT_SIPSIG": "syscall", - "syscall.IFT_SIPTG": "syscall", - "syscall.IFT_SLIP": "syscall", - "syscall.IFT_SMDSDXI": "syscall", - "syscall.IFT_SMDSICIP": "syscall", - "syscall.IFT_SONET": "syscall", - "syscall.IFT_SONETOVERHEADCHANNEL": "syscall", - "syscall.IFT_SONETPATH": "syscall", - "syscall.IFT_SONETVT": "syscall", - "syscall.IFT_SRP": "syscall", - "syscall.IFT_SS7SIGLINK": "syscall", - "syscall.IFT_STACKTOSTACK": "syscall", - "syscall.IFT_STARLAN": "syscall", - "syscall.IFT_STF": "syscall", - "syscall.IFT_T1": "syscall", - "syscall.IFT_TDLC": "syscall", - "syscall.IFT_TELINK": "syscall", - "syscall.IFT_TERMPAD": "syscall", - "syscall.IFT_TR008": "syscall", - "syscall.IFT_TRANSPHDLC": "syscall", - "syscall.IFT_TUNNEL": "syscall", - "syscall.IFT_ULTRA": "syscall", - "syscall.IFT_USB": "syscall", - "syscall.IFT_V11": "syscall", - "syscall.IFT_V35": "syscall", - "syscall.IFT_V36": "syscall", - "syscall.IFT_V37": "syscall", - "syscall.IFT_VDSL": "syscall", - "syscall.IFT_VIRTUALIPADDRESS": "syscall", - "syscall.IFT_VIRTUALTG": "syscall", - "syscall.IFT_VOICEDID": "syscall", - "syscall.IFT_VOICEEM": "syscall", - "syscall.IFT_VOICEEMFGD": "syscall", - "syscall.IFT_VOICEENCAP": "syscall", - "syscall.IFT_VOICEFGDEANA": "syscall", - "syscall.IFT_VOICEFXO": "syscall", - "syscall.IFT_VOICEFXS": "syscall", - "syscall.IFT_VOICEOVERATM": "syscall", - "syscall.IFT_VOICEOVERCABLE": "syscall", - "syscall.IFT_VOICEOVERFRAMERELAY": "syscall", - "syscall.IFT_VOICEOVERIP": "syscall", - "syscall.IFT_X213": "syscall", - "syscall.IFT_X25": "syscall", - "syscall.IFT_X25DDN": "syscall", - "syscall.IFT_X25HUNTGROUP": "syscall", - "syscall.IFT_X25MLP": "syscall", - "syscall.IFT_X25PLE": "syscall", - "syscall.IFT_XETHER": "syscall", - "syscall.IGNBRK": "syscall", - "syscall.IGNCR": "syscall", - "syscall.IGNORE": "syscall", - "syscall.IGNPAR": "syscall", - "syscall.IMAXBEL": "syscall", - "syscall.INFINITE": "syscall", - "syscall.INLCR": "syscall", - "syscall.INPCK": "syscall", - "syscall.INVALID_FILE_ATTRIBUTES": "syscall", - "syscall.IN_ACCESS": "syscall", - "syscall.IN_ALL_EVENTS": "syscall", - "syscall.IN_ATTRIB": "syscall", - "syscall.IN_CLASSA_HOST": "syscall", - "syscall.IN_CLASSA_MAX": "syscall", - "syscall.IN_CLASSA_NET": "syscall", - "syscall.IN_CLASSA_NSHIFT": "syscall", - "syscall.IN_CLASSB_HOST": "syscall", - "syscall.IN_CLASSB_MAX": "syscall", - "syscall.IN_CLASSB_NET": "syscall", - "syscall.IN_CLASSB_NSHIFT": "syscall", - "syscall.IN_CLASSC_HOST": "syscall", - "syscall.IN_CLASSC_NET": "syscall", - "syscall.IN_CLASSC_NSHIFT": "syscall", - "syscall.IN_CLASSD_HOST": "syscall", - "syscall.IN_CLASSD_NET": "syscall", - "syscall.IN_CLASSD_NSHIFT": "syscall", - "syscall.IN_CLOEXEC": "syscall", - "syscall.IN_CLOSE": "syscall", - "syscall.IN_CLOSE_NOWRITE": "syscall", - "syscall.IN_CLOSE_WRITE": "syscall", - "syscall.IN_CREATE": "syscall", - "syscall.IN_DELETE": "syscall", - "syscall.IN_DELETE_SELF": "syscall", - "syscall.IN_DONT_FOLLOW": "syscall", - "syscall.IN_EXCL_UNLINK": "syscall", - "syscall.IN_IGNORED": "syscall", - "syscall.IN_ISDIR": "syscall", - "syscall.IN_LINKLOCALNETNUM": "syscall", - "syscall.IN_LOOPBACKNET": "syscall", - "syscall.IN_MASK_ADD": "syscall", - "syscall.IN_MODIFY": "syscall", - "syscall.IN_MOVE": "syscall", - "syscall.IN_MOVED_FROM": "syscall", - "syscall.IN_MOVED_TO": "syscall", - "syscall.IN_MOVE_SELF": "syscall", - "syscall.IN_NONBLOCK": "syscall", - "syscall.IN_ONESHOT": "syscall", - "syscall.IN_ONLYDIR": "syscall", - "syscall.IN_OPEN": "syscall", - "syscall.IN_Q_OVERFLOW": "syscall", - "syscall.IN_RFC3021_HOST": "syscall", - "syscall.IN_RFC3021_MASK": "syscall", - "syscall.IN_RFC3021_NET": "syscall", - "syscall.IN_RFC3021_NSHIFT": "syscall", - "syscall.IN_UNMOUNT": "syscall", - "syscall.IOC_IN": "syscall", - "syscall.IOC_INOUT": "syscall", - "syscall.IOC_OUT": "syscall", - "syscall.IOC_VENDOR": "syscall", - "syscall.IOC_WS2": "syscall", - "syscall.IO_REPARSE_TAG_SYMLINK": "syscall", - "syscall.IPMreq": "syscall", - "syscall.IPMreqn": "syscall", - "syscall.IPPROTO_3PC": "syscall", - "syscall.IPPROTO_ADFS": "syscall", - "syscall.IPPROTO_AH": "syscall", - "syscall.IPPROTO_AHIP": "syscall", - "syscall.IPPROTO_APES": "syscall", - "syscall.IPPROTO_ARGUS": "syscall", - "syscall.IPPROTO_AX25": "syscall", - "syscall.IPPROTO_BHA": "syscall", - "syscall.IPPROTO_BLT": "syscall", - "syscall.IPPROTO_BRSATMON": "syscall", - "syscall.IPPROTO_CARP": "syscall", - "syscall.IPPROTO_CFTP": "syscall", - "syscall.IPPROTO_CHAOS": "syscall", - "syscall.IPPROTO_CMTP": "syscall", - "syscall.IPPROTO_COMP": "syscall", - "syscall.IPPROTO_CPHB": "syscall", - "syscall.IPPROTO_CPNX": "syscall", - "syscall.IPPROTO_DCCP": "syscall", - "syscall.IPPROTO_DDP": "syscall", - "syscall.IPPROTO_DGP": "syscall", - "syscall.IPPROTO_DIVERT": "syscall", - "syscall.IPPROTO_DIVERT_INIT": "syscall", - "syscall.IPPROTO_DIVERT_RESP": "syscall", - "syscall.IPPROTO_DONE": "syscall", - "syscall.IPPROTO_DSTOPTS": "syscall", - "syscall.IPPROTO_EGP": "syscall", - "syscall.IPPROTO_EMCON": "syscall", - "syscall.IPPROTO_ENCAP": "syscall", - "syscall.IPPROTO_EON": "syscall", - "syscall.IPPROTO_ESP": "syscall", - "syscall.IPPROTO_ETHERIP": "syscall", - "syscall.IPPROTO_FRAGMENT": "syscall", - "syscall.IPPROTO_GGP": "syscall", - "syscall.IPPROTO_GMTP": "syscall", - "syscall.IPPROTO_GRE": "syscall", - "syscall.IPPROTO_HELLO": "syscall", - "syscall.IPPROTO_HMP": "syscall", - "syscall.IPPROTO_HOPOPTS": "syscall", - "syscall.IPPROTO_ICMP": "syscall", - "syscall.IPPROTO_ICMPV6": "syscall", - "syscall.IPPROTO_IDP": "syscall", - "syscall.IPPROTO_IDPR": "syscall", - "syscall.IPPROTO_IDRP": "syscall", - "syscall.IPPROTO_IGMP": "syscall", - "syscall.IPPROTO_IGP": "syscall", - "syscall.IPPROTO_IGRP": "syscall", - "syscall.IPPROTO_IL": "syscall", - "syscall.IPPROTO_INLSP": "syscall", - "syscall.IPPROTO_INP": "syscall", - "syscall.IPPROTO_IP": "syscall", - "syscall.IPPROTO_IPCOMP": "syscall", - "syscall.IPPROTO_IPCV": "syscall", - "syscall.IPPROTO_IPEIP": "syscall", - "syscall.IPPROTO_IPIP": "syscall", - "syscall.IPPROTO_IPPC": "syscall", - "syscall.IPPROTO_IPV4": "syscall", - "syscall.IPPROTO_IPV6": "syscall", - "syscall.IPPROTO_IPV6_ICMP": "syscall", - "syscall.IPPROTO_IRTP": "syscall", - "syscall.IPPROTO_KRYPTOLAN": "syscall", - "syscall.IPPROTO_LARP": "syscall", - "syscall.IPPROTO_LEAF1": "syscall", - "syscall.IPPROTO_LEAF2": "syscall", - "syscall.IPPROTO_MAX": "syscall", - "syscall.IPPROTO_MAXID": "syscall", - "syscall.IPPROTO_MEAS": "syscall", - "syscall.IPPROTO_MH": "syscall", - "syscall.IPPROTO_MHRP": "syscall", - "syscall.IPPROTO_MICP": "syscall", - "syscall.IPPROTO_MOBILE": "syscall", - "syscall.IPPROTO_MPLS": "syscall", - "syscall.IPPROTO_MTP": "syscall", - "syscall.IPPROTO_MUX": "syscall", - "syscall.IPPROTO_ND": "syscall", - "syscall.IPPROTO_NHRP": "syscall", - "syscall.IPPROTO_NONE": "syscall", - "syscall.IPPROTO_NSP": "syscall", - "syscall.IPPROTO_NVPII": "syscall", - "syscall.IPPROTO_OLD_DIVERT": "syscall", - "syscall.IPPROTO_OSPFIGP": "syscall", - "syscall.IPPROTO_PFSYNC": "syscall", - "syscall.IPPROTO_PGM": "syscall", - "syscall.IPPROTO_PIGP": "syscall", - "syscall.IPPROTO_PIM": "syscall", - "syscall.IPPROTO_PRM": "syscall", - "syscall.IPPROTO_PUP": "syscall", - "syscall.IPPROTO_PVP": "syscall", - "syscall.IPPROTO_RAW": "syscall", - "syscall.IPPROTO_RCCMON": "syscall", - "syscall.IPPROTO_RDP": "syscall", - "syscall.IPPROTO_ROUTING": "syscall", - "syscall.IPPROTO_RSVP": "syscall", - "syscall.IPPROTO_RVD": "syscall", - "syscall.IPPROTO_SATEXPAK": "syscall", - "syscall.IPPROTO_SATMON": "syscall", - "syscall.IPPROTO_SCCSP": "syscall", - "syscall.IPPROTO_SCTP": "syscall", - "syscall.IPPROTO_SDRP": "syscall", - "syscall.IPPROTO_SEND": "syscall", - "syscall.IPPROTO_SEP": "syscall", - "syscall.IPPROTO_SKIP": "syscall", - "syscall.IPPROTO_SPACER": "syscall", - "syscall.IPPROTO_SRPC": "syscall", - "syscall.IPPROTO_ST": "syscall", - "syscall.IPPROTO_SVMTP": "syscall", - "syscall.IPPROTO_SWIPE": "syscall", - "syscall.IPPROTO_TCF": "syscall", - "syscall.IPPROTO_TCP": "syscall", - "syscall.IPPROTO_TLSP": "syscall", - "syscall.IPPROTO_TP": "syscall", - "syscall.IPPROTO_TPXX": "syscall", - "syscall.IPPROTO_TRUNK1": "syscall", - "syscall.IPPROTO_TRUNK2": "syscall", - "syscall.IPPROTO_TTP": "syscall", - "syscall.IPPROTO_UDP": "syscall", - "syscall.IPPROTO_UDPLITE": "syscall", - "syscall.IPPROTO_VINES": "syscall", - "syscall.IPPROTO_VISA": "syscall", - "syscall.IPPROTO_VMTP": "syscall", - "syscall.IPPROTO_VRRP": "syscall", - "syscall.IPPROTO_WBEXPAK": "syscall", - "syscall.IPPROTO_WBMON": "syscall", - "syscall.IPPROTO_WSN": "syscall", - "syscall.IPPROTO_XNET": "syscall", - "syscall.IPPROTO_XTP": "syscall", - "syscall.IPV6_2292DSTOPTS": "syscall", - "syscall.IPV6_2292HOPLIMIT": "syscall", - "syscall.IPV6_2292HOPOPTS": "syscall", - "syscall.IPV6_2292NEXTHOP": "syscall", - "syscall.IPV6_2292PKTINFO": "syscall", - "syscall.IPV6_2292PKTOPTIONS": "syscall", - "syscall.IPV6_2292RTHDR": "syscall", - "syscall.IPV6_ADDRFORM": "syscall", - "syscall.IPV6_ADD_MEMBERSHIP": "syscall", - "syscall.IPV6_AUTHHDR": "syscall", - "syscall.IPV6_AUTH_LEVEL": "syscall", - "syscall.IPV6_AUTOFLOWLABEL": "syscall", - "syscall.IPV6_BINDANY": "syscall", - "syscall.IPV6_BINDV6ONLY": "syscall", - "syscall.IPV6_BOUND_IF": "syscall", - "syscall.IPV6_CHECKSUM": "syscall", - "syscall.IPV6_DEFAULT_MULTICAST_HOPS": "syscall", - "syscall.IPV6_DEFAULT_MULTICAST_LOOP": "syscall", - "syscall.IPV6_DEFHLIM": "syscall", - "syscall.IPV6_DONTFRAG": "syscall", - "syscall.IPV6_DROP_MEMBERSHIP": "syscall", - "syscall.IPV6_DSTOPTS": "syscall", - "syscall.IPV6_ESP_NETWORK_LEVEL": "syscall", - "syscall.IPV6_ESP_TRANS_LEVEL": "syscall", - "syscall.IPV6_FAITH": "syscall", - "syscall.IPV6_FLOWINFO_MASK": "syscall", - "syscall.IPV6_FLOWLABEL_MASK": "syscall", - "syscall.IPV6_FRAGTTL": "syscall", - "syscall.IPV6_FW_ADD": "syscall", - "syscall.IPV6_FW_DEL": "syscall", - "syscall.IPV6_FW_FLUSH": "syscall", - "syscall.IPV6_FW_GET": "syscall", - "syscall.IPV6_FW_ZERO": "syscall", - "syscall.IPV6_HLIMDEC": "syscall", - "syscall.IPV6_HOPLIMIT": "syscall", - "syscall.IPV6_HOPOPTS": "syscall", - "syscall.IPV6_IPCOMP_LEVEL": "syscall", - "syscall.IPV6_IPSEC_POLICY": "syscall", - "syscall.IPV6_JOIN_ANYCAST": "syscall", - "syscall.IPV6_JOIN_GROUP": "syscall", - "syscall.IPV6_LEAVE_ANYCAST": "syscall", - "syscall.IPV6_LEAVE_GROUP": "syscall", - "syscall.IPV6_MAXHLIM": "syscall", - "syscall.IPV6_MAXOPTHDR": "syscall", - "syscall.IPV6_MAXPACKET": "syscall", - "syscall.IPV6_MAX_GROUP_SRC_FILTER": "syscall", - "syscall.IPV6_MAX_MEMBERSHIPS": "syscall", - "syscall.IPV6_MAX_SOCK_SRC_FILTER": "syscall", - "syscall.IPV6_MIN_MEMBERSHIPS": "syscall", - "syscall.IPV6_MMTU": "syscall", - "syscall.IPV6_MSFILTER": "syscall", - "syscall.IPV6_MTU": "syscall", - "syscall.IPV6_MTU_DISCOVER": "syscall", - "syscall.IPV6_MULTICAST_HOPS": "syscall", - "syscall.IPV6_MULTICAST_IF": "syscall", - "syscall.IPV6_MULTICAST_LOOP": "syscall", - "syscall.IPV6_NEXTHOP": "syscall", - "syscall.IPV6_OPTIONS": "syscall", - "syscall.IPV6_PATHMTU": "syscall", - "syscall.IPV6_PIPEX": "syscall", - "syscall.IPV6_PKTINFO": "syscall", - "syscall.IPV6_PMTUDISC_DO": "syscall", - "syscall.IPV6_PMTUDISC_DONT": "syscall", - "syscall.IPV6_PMTUDISC_PROBE": "syscall", - "syscall.IPV6_PMTUDISC_WANT": "syscall", - "syscall.IPV6_PORTRANGE": "syscall", - "syscall.IPV6_PORTRANGE_DEFAULT": "syscall", - "syscall.IPV6_PORTRANGE_HIGH": "syscall", - "syscall.IPV6_PORTRANGE_LOW": "syscall", - "syscall.IPV6_PREFER_TEMPADDR": "syscall", - "syscall.IPV6_RECVDSTOPTS": "syscall", - "syscall.IPV6_RECVDSTPORT": "syscall", - "syscall.IPV6_RECVERR": "syscall", - "syscall.IPV6_RECVHOPLIMIT": "syscall", - "syscall.IPV6_RECVHOPOPTS": "syscall", - "syscall.IPV6_RECVPATHMTU": "syscall", - "syscall.IPV6_RECVPKTINFO": "syscall", - "syscall.IPV6_RECVRTHDR": "syscall", - "syscall.IPV6_RECVTCLASS": "syscall", - "syscall.IPV6_ROUTER_ALERT": "syscall", - "syscall.IPV6_RTABLE": "syscall", - "syscall.IPV6_RTHDR": "syscall", - "syscall.IPV6_RTHDRDSTOPTS": "syscall", - "syscall.IPV6_RTHDR_LOOSE": "syscall", - "syscall.IPV6_RTHDR_STRICT": "syscall", - "syscall.IPV6_RTHDR_TYPE_0": "syscall", - "syscall.IPV6_RXDSTOPTS": "syscall", - "syscall.IPV6_RXHOPOPTS": "syscall", - "syscall.IPV6_SOCKOPT_RESERVED1": "syscall", - "syscall.IPV6_TCLASS": "syscall", - "syscall.IPV6_UNICAST_HOPS": "syscall", - "syscall.IPV6_USE_MIN_MTU": "syscall", - "syscall.IPV6_V6ONLY": "syscall", - "syscall.IPV6_VERSION": "syscall", - "syscall.IPV6_VERSION_MASK": "syscall", - "syscall.IPV6_XFRM_POLICY": "syscall", - "syscall.IP_ADD_MEMBERSHIP": "syscall", - "syscall.IP_ADD_SOURCE_MEMBERSHIP": "syscall", - "syscall.IP_AUTH_LEVEL": "syscall", - "syscall.IP_BINDANY": "syscall", - "syscall.IP_BLOCK_SOURCE": "syscall", - "syscall.IP_BOUND_IF": "syscall", - "syscall.IP_DEFAULT_MULTICAST_LOOP": "syscall", - "syscall.IP_DEFAULT_MULTICAST_TTL": "syscall", - "syscall.IP_DF": "syscall", - "syscall.IP_DIVERTFL": "syscall", - "syscall.IP_DONTFRAG": "syscall", - "syscall.IP_DROP_MEMBERSHIP": "syscall", - "syscall.IP_DROP_SOURCE_MEMBERSHIP": "syscall", - "syscall.IP_DUMMYNET3": "syscall", - "syscall.IP_DUMMYNET_CONFIGURE": "syscall", - "syscall.IP_DUMMYNET_DEL": "syscall", - "syscall.IP_DUMMYNET_FLUSH": "syscall", - "syscall.IP_DUMMYNET_GET": "syscall", - "syscall.IP_EF": "syscall", - "syscall.IP_ERRORMTU": "syscall", - "syscall.IP_ESP_NETWORK_LEVEL": "syscall", - "syscall.IP_ESP_TRANS_LEVEL": "syscall", - "syscall.IP_FAITH": "syscall", - "syscall.IP_FREEBIND": "syscall", - "syscall.IP_FW3": "syscall", - "syscall.IP_FW_ADD": "syscall", - "syscall.IP_FW_DEL": "syscall", - "syscall.IP_FW_FLUSH": "syscall", - "syscall.IP_FW_GET": "syscall", - "syscall.IP_FW_NAT_CFG": "syscall", - "syscall.IP_FW_NAT_DEL": "syscall", - "syscall.IP_FW_NAT_GET_CONFIG": "syscall", - "syscall.IP_FW_NAT_GET_LOG": "syscall", - "syscall.IP_FW_RESETLOG": "syscall", - "syscall.IP_FW_TABLE_ADD": "syscall", - "syscall.IP_FW_TABLE_DEL": "syscall", - "syscall.IP_FW_TABLE_FLUSH": "syscall", - "syscall.IP_FW_TABLE_GETSIZE": "syscall", - "syscall.IP_FW_TABLE_LIST": "syscall", - "syscall.IP_FW_ZERO": "syscall", - "syscall.IP_HDRINCL": "syscall", - "syscall.IP_IPCOMP_LEVEL": "syscall", - "syscall.IP_IPSECFLOWINFO": "syscall", - "syscall.IP_IPSEC_LOCAL_AUTH": "syscall", - "syscall.IP_IPSEC_LOCAL_CRED": "syscall", - "syscall.IP_IPSEC_LOCAL_ID": "syscall", - "syscall.IP_IPSEC_POLICY": "syscall", - "syscall.IP_IPSEC_REMOTE_AUTH": "syscall", - "syscall.IP_IPSEC_REMOTE_CRED": "syscall", - "syscall.IP_IPSEC_REMOTE_ID": "syscall", - "syscall.IP_MAXPACKET": "syscall", - "syscall.IP_MAX_GROUP_SRC_FILTER": "syscall", - "syscall.IP_MAX_MEMBERSHIPS": "syscall", - "syscall.IP_MAX_SOCK_MUTE_FILTER": "syscall", - "syscall.IP_MAX_SOCK_SRC_FILTER": "syscall", - "syscall.IP_MAX_SOURCE_FILTER": "syscall", - "syscall.IP_MF": "syscall", - "syscall.IP_MINFRAGSIZE": "syscall", - "syscall.IP_MINTTL": "syscall", - "syscall.IP_MIN_MEMBERSHIPS": "syscall", - "syscall.IP_MSFILTER": "syscall", - "syscall.IP_MSS": "syscall", - "syscall.IP_MTU": "syscall", - "syscall.IP_MTU_DISCOVER": "syscall", - "syscall.IP_MULTICAST_IF": "syscall", - "syscall.IP_MULTICAST_IFINDEX": "syscall", - "syscall.IP_MULTICAST_LOOP": "syscall", - "syscall.IP_MULTICAST_TTL": "syscall", - "syscall.IP_MULTICAST_VIF": "syscall", - "syscall.IP_NAT__XXX": "syscall", - "syscall.IP_OFFMASK": "syscall", - "syscall.IP_OLD_FW_ADD": "syscall", - "syscall.IP_OLD_FW_DEL": "syscall", - "syscall.IP_OLD_FW_FLUSH": "syscall", - "syscall.IP_OLD_FW_GET": "syscall", - "syscall.IP_OLD_FW_RESETLOG": "syscall", - "syscall.IP_OLD_FW_ZERO": "syscall", - "syscall.IP_ONESBCAST": "syscall", - "syscall.IP_OPTIONS": "syscall", - "syscall.IP_ORIGDSTADDR": "syscall", - "syscall.IP_PASSSEC": "syscall", - "syscall.IP_PIPEX": "syscall", - "syscall.IP_PKTINFO": "syscall", - "syscall.IP_PKTOPTIONS": "syscall", - "syscall.IP_PMTUDISC": "syscall", - "syscall.IP_PMTUDISC_DO": "syscall", - "syscall.IP_PMTUDISC_DONT": "syscall", - "syscall.IP_PMTUDISC_PROBE": "syscall", - "syscall.IP_PMTUDISC_WANT": "syscall", - "syscall.IP_PORTRANGE": "syscall", - "syscall.IP_PORTRANGE_DEFAULT": "syscall", - "syscall.IP_PORTRANGE_HIGH": "syscall", - "syscall.IP_PORTRANGE_LOW": "syscall", - "syscall.IP_RECVDSTADDR": "syscall", - "syscall.IP_RECVDSTPORT": "syscall", - "syscall.IP_RECVERR": "syscall", - "syscall.IP_RECVIF": "syscall", - "syscall.IP_RECVOPTS": "syscall", - "syscall.IP_RECVORIGDSTADDR": "syscall", - "syscall.IP_RECVPKTINFO": "syscall", - "syscall.IP_RECVRETOPTS": "syscall", - "syscall.IP_RECVRTABLE": "syscall", - "syscall.IP_RECVTOS": "syscall", - "syscall.IP_RECVTTL": "syscall", - "syscall.IP_RETOPTS": "syscall", - "syscall.IP_RF": "syscall", - "syscall.IP_ROUTER_ALERT": "syscall", - "syscall.IP_RSVP_OFF": "syscall", - "syscall.IP_RSVP_ON": "syscall", - "syscall.IP_RSVP_VIF_OFF": "syscall", - "syscall.IP_RSVP_VIF_ON": "syscall", - "syscall.IP_RTABLE": "syscall", - "syscall.IP_SENDSRCADDR": "syscall", - "syscall.IP_STRIPHDR": "syscall", - "syscall.IP_TOS": "syscall", - "syscall.IP_TRAFFIC_MGT_BACKGROUND": "syscall", - "syscall.IP_TRANSPARENT": "syscall", - "syscall.IP_TTL": "syscall", - "syscall.IP_UNBLOCK_SOURCE": "syscall", - "syscall.IP_XFRM_POLICY": "syscall", - "syscall.IPv6MTUInfo": "syscall", - "syscall.IPv6Mreq": "syscall", - "syscall.ISIG": "syscall", - "syscall.ISTRIP": "syscall", - "syscall.IUCLC": "syscall", - "syscall.IUTF8": "syscall", - "syscall.IXANY": "syscall", - "syscall.IXOFF": "syscall", - "syscall.IXON": "syscall", - "syscall.IfAddrmsg": "syscall", - "syscall.IfAnnounceMsghdr": "syscall", - "syscall.IfData": "syscall", - "syscall.IfInfomsg": "syscall", - "syscall.IfMsghdr": "syscall", - "syscall.IfaMsghdr": "syscall", - "syscall.IfmaMsghdr": "syscall", - "syscall.IfmaMsghdr2": "syscall", - "syscall.ImplementsGetwd": "syscall", - "syscall.Inet4Pktinfo": "syscall", - "syscall.Inet6Pktinfo": "syscall", - "syscall.InotifyAddWatch": "syscall", - "syscall.InotifyEvent": "syscall", - "syscall.InotifyInit": "syscall", - "syscall.InotifyInit1": "syscall", - "syscall.InotifyRmWatch": "syscall", - "syscall.InterfaceAddrMessage": "syscall", - "syscall.InterfaceAnnounceMessage": "syscall", - "syscall.InterfaceInfo": "syscall", - "syscall.InterfaceMessage": "syscall", - "syscall.InterfaceMulticastAddrMessage": "syscall", - "syscall.InvalidHandle": "syscall", - "syscall.Ioperm": "syscall", - "syscall.Iopl": "syscall", - "syscall.Iovec": "syscall", - "syscall.IpAdapterInfo": "syscall", - "syscall.IpAddrString": "syscall", - "syscall.IpAddressString": "syscall", - "syscall.IpMaskString": "syscall", - "syscall.Issetugid": "syscall", - "syscall.KEY_ALL_ACCESS": "syscall", - "syscall.KEY_CREATE_LINK": "syscall", - "syscall.KEY_CREATE_SUB_KEY": "syscall", - "syscall.KEY_ENUMERATE_SUB_KEYS": "syscall", - "syscall.KEY_EXECUTE": "syscall", - "syscall.KEY_NOTIFY": "syscall", - "syscall.KEY_QUERY_VALUE": "syscall", - "syscall.KEY_READ": "syscall", - "syscall.KEY_SET_VALUE": "syscall", - "syscall.KEY_WOW64_32KEY": "syscall", - "syscall.KEY_WOW64_64KEY": "syscall", - "syscall.KEY_WRITE": "syscall", - "syscall.Kevent": "syscall", - "syscall.Kevent_t": "syscall", - "syscall.Kill": "syscall", - "syscall.Klogctl": "syscall", - "syscall.Kqueue": "syscall", - "syscall.LANG_ENGLISH": "syscall", - "syscall.LAYERED_PROTOCOL": "syscall", - "syscall.LCNT_OVERLOAD_FLUSH": "syscall", - "syscall.LINUX_REBOOT_CMD_CAD_OFF": "syscall", - "syscall.LINUX_REBOOT_CMD_CAD_ON": "syscall", - "syscall.LINUX_REBOOT_CMD_HALT": "syscall", - "syscall.LINUX_REBOOT_CMD_KEXEC": "syscall", - "syscall.LINUX_REBOOT_CMD_POWER_OFF": "syscall", - "syscall.LINUX_REBOOT_CMD_RESTART": "syscall", - "syscall.LINUX_REBOOT_CMD_RESTART2": "syscall", - "syscall.LINUX_REBOOT_CMD_SW_SUSPEND": "syscall", - "syscall.LINUX_REBOOT_MAGIC1": "syscall", - "syscall.LINUX_REBOOT_MAGIC2": "syscall", - "syscall.LOCK_EX": "syscall", - "syscall.LOCK_NB": "syscall", - "syscall.LOCK_SH": "syscall", - "syscall.LOCK_UN": "syscall", - "syscall.LazyDLL": "syscall", - "syscall.LazyProc": "syscall", - "syscall.Lchown": "syscall", - "syscall.Linger": "syscall", - "syscall.Link": "syscall", - "syscall.Listen": "syscall", - "syscall.Listxattr": "syscall", - "syscall.LoadCancelIoEx": "syscall", - "syscall.LoadConnectEx": "syscall", - "syscall.LoadCreateSymbolicLink": "syscall", - "syscall.LoadDLL": "syscall", - "syscall.LoadGetAddrInfo": "syscall", - "syscall.LoadLibrary": "syscall", - "syscall.LoadSetFileCompletionNotificationModes": "syscall", - "syscall.LocalFree": "syscall", - "syscall.Log2phys_t": "syscall", - "syscall.LookupAccountName": "syscall", - "syscall.LookupAccountSid": "syscall", - "syscall.LookupSID": "syscall", - "syscall.LsfJump": "syscall", - "syscall.LsfSocket": "syscall", - "syscall.LsfStmt": "syscall", - "syscall.Lstat": "syscall", - "syscall.MADV_AUTOSYNC": "syscall", - "syscall.MADV_CAN_REUSE": "syscall", - "syscall.MADV_CORE": "syscall", - "syscall.MADV_DOFORK": "syscall", - "syscall.MADV_DONTFORK": "syscall", - "syscall.MADV_DONTNEED": "syscall", - "syscall.MADV_FREE": "syscall", - "syscall.MADV_FREE_REUSABLE": "syscall", - "syscall.MADV_FREE_REUSE": "syscall", - "syscall.MADV_HUGEPAGE": "syscall", - "syscall.MADV_HWPOISON": "syscall", - "syscall.MADV_MERGEABLE": "syscall", - "syscall.MADV_NOCORE": "syscall", - "syscall.MADV_NOHUGEPAGE": "syscall", - "syscall.MADV_NORMAL": "syscall", - "syscall.MADV_NOSYNC": "syscall", - "syscall.MADV_PROTECT": "syscall", - "syscall.MADV_RANDOM": "syscall", - "syscall.MADV_REMOVE": "syscall", - "syscall.MADV_SEQUENTIAL": "syscall", - "syscall.MADV_SPACEAVAIL": "syscall", - "syscall.MADV_UNMERGEABLE": "syscall", - "syscall.MADV_WILLNEED": "syscall", - "syscall.MADV_ZERO_WIRED_PAGES": "syscall", - "syscall.MAP_32BIT": "syscall", - "syscall.MAP_ALIGNED_SUPER": "syscall", - "syscall.MAP_ALIGNMENT_16MB": "syscall", - "syscall.MAP_ALIGNMENT_1TB": "syscall", - "syscall.MAP_ALIGNMENT_256TB": "syscall", - "syscall.MAP_ALIGNMENT_4GB": "syscall", - "syscall.MAP_ALIGNMENT_64KB": "syscall", - "syscall.MAP_ALIGNMENT_64PB": "syscall", - "syscall.MAP_ALIGNMENT_MASK": "syscall", - "syscall.MAP_ALIGNMENT_SHIFT": "syscall", - "syscall.MAP_ANON": "syscall", - "syscall.MAP_ANONYMOUS": "syscall", - "syscall.MAP_COPY": "syscall", - "syscall.MAP_DENYWRITE": "syscall", - "syscall.MAP_EXECUTABLE": "syscall", - "syscall.MAP_FILE": "syscall", - "syscall.MAP_FIXED": "syscall", - "syscall.MAP_FLAGMASK": "syscall", - "syscall.MAP_GROWSDOWN": "syscall", - "syscall.MAP_HASSEMAPHORE": "syscall", - "syscall.MAP_HUGETLB": "syscall", - "syscall.MAP_INHERIT": "syscall", - "syscall.MAP_INHERIT_COPY": "syscall", - "syscall.MAP_INHERIT_DEFAULT": "syscall", - "syscall.MAP_INHERIT_DONATE_COPY": "syscall", - "syscall.MAP_INHERIT_NONE": "syscall", - "syscall.MAP_INHERIT_SHARE": "syscall", - "syscall.MAP_JIT": "syscall", - "syscall.MAP_LOCKED": "syscall", - "syscall.MAP_NOCACHE": "syscall", - "syscall.MAP_NOCORE": "syscall", - "syscall.MAP_NOEXTEND": "syscall", - "syscall.MAP_NONBLOCK": "syscall", - "syscall.MAP_NORESERVE": "syscall", - "syscall.MAP_NOSYNC": "syscall", - "syscall.MAP_POPULATE": "syscall", - "syscall.MAP_PREFAULT_READ": "syscall", - "syscall.MAP_PRIVATE": "syscall", - "syscall.MAP_RENAME": "syscall", - "syscall.MAP_RESERVED0080": "syscall", - "syscall.MAP_RESERVED0100": "syscall", - "syscall.MAP_SHARED": "syscall", - "syscall.MAP_STACK": "syscall", - "syscall.MAP_TRYFIXED": "syscall", - "syscall.MAP_TYPE": "syscall", - "syscall.MAP_WIRED": "syscall", - "syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE": "syscall", - "syscall.MAXLEN_IFDESCR": "syscall", - "syscall.MAXLEN_PHYSADDR": "syscall", - "syscall.MAX_ADAPTER_ADDRESS_LENGTH": "syscall", - "syscall.MAX_ADAPTER_DESCRIPTION_LENGTH": "syscall", - "syscall.MAX_ADAPTER_NAME_LENGTH": "syscall", - "syscall.MAX_COMPUTERNAME_LENGTH": "syscall", - "syscall.MAX_INTERFACE_NAME_LEN": "syscall", - "syscall.MAX_LONG_PATH": "syscall", - "syscall.MAX_PATH": "syscall", - "syscall.MAX_PROTOCOL_CHAIN": "syscall", - "syscall.MCL_CURRENT": "syscall", - "syscall.MCL_FUTURE": "syscall", - "syscall.MNT_DETACH": "syscall", - "syscall.MNT_EXPIRE": "syscall", - "syscall.MNT_FORCE": "syscall", - "syscall.MSG_BCAST": "syscall", - "syscall.MSG_CMSG_CLOEXEC": "syscall", - "syscall.MSG_COMPAT": "syscall", - "syscall.MSG_CONFIRM": "syscall", - "syscall.MSG_CONTROLMBUF": "syscall", - "syscall.MSG_CTRUNC": "syscall", - "syscall.MSG_DONTROUTE": "syscall", - "syscall.MSG_DONTWAIT": "syscall", - "syscall.MSG_EOF": "syscall", - "syscall.MSG_EOR": "syscall", - "syscall.MSG_ERRQUEUE": "syscall", - "syscall.MSG_FASTOPEN": "syscall", - "syscall.MSG_FIN": "syscall", - "syscall.MSG_FLUSH": "syscall", - "syscall.MSG_HAVEMORE": "syscall", - "syscall.MSG_HOLD": "syscall", - "syscall.MSG_IOVUSRSPACE": "syscall", - "syscall.MSG_LENUSRSPACE": "syscall", - "syscall.MSG_MCAST": "syscall", - "syscall.MSG_MORE": "syscall", - "syscall.MSG_NAMEMBUF": "syscall", - "syscall.MSG_NBIO": "syscall", - "syscall.MSG_NEEDSA": "syscall", - "syscall.MSG_NOSIGNAL": "syscall", - "syscall.MSG_NOTIFICATION": "syscall", - "syscall.MSG_OOB": "syscall", - "syscall.MSG_PEEK": "syscall", - "syscall.MSG_PROXY": "syscall", - "syscall.MSG_RCVMORE": "syscall", - "syscall.MSG_RST": "syscall", - "syscall.MSG_SEND": "syscall", - "syscall.MSG_SYN": "syscall", - "syscall.MSG_TRUNC": "syscall", - "syscall.MSG_TRYHARD": "syscall", - "syscall.MSG_USERFLAGS": "syscall", - "syscall.MSG_WAITALL": "syscall", - "syscall.MSG_WAITFORONE": "syscall", - "syscall.MSG_WAITSTREAM": "syscall", - "syscall.MS_ACTIVE": "syscall", - "syscall.MS_ASYNC": "syscall", - "syscall.MS_BIND": "syscall", - "syscall.MS_DEACTIVATE": "syscall", - "syscall.MS_DIRSYNC": "syscall", - "syscall.MS_INVALIDATE": "syscall", - "syscall.MS_I_VERSION": "syscall", - "syscall.MS_KERNMOUNT": "syscall", - "syscall.MS_KILLPAGES": "syscall", - "syscall.MS_MANDLOCK": "syscall", - "syscall.MS_MGC_MSK": "syscall", - "syscall.MS_MGC_VAL": "syscall", - "syscall.MS_MOVE": "syscall", - "syscall.MS_NOATIME": "syscall", - "syscall.MS_NODEV": "syscall", - "syscall.MS_NODIRATIME": "syscall", - "syscall.MS_NOEXEC": "syscall", - "syscall.MS_NOSUID": "syscall", - "syscall.MS_NOUSER": "syscall", - "syscall.MS_POSIXACL": "syscall", - "syscall.MS_PRIVATE": "syscall", - "syscall.MS_RDONLY": "syscall", - "syscall.MS_REC": "syscall", - "syscall.MS_RELATIME": "syscall", - "syscall.MS_REMOUNT": "syscall", - "syscall.MS_RMT_MASK": "syscall", - "syscall.MS_SHARED": "syscall", - "syscall.MS_SILENT": "syscall", - "syscall.MS_SLAVE": "syscall", - "syscall.MS_STRICTATIME": "syscall", - "syscall.MS_SYNC": "syscall", - "syscall.MS_SYNCHRONOUS": "syscall", - "syscall.MS_UNBINDABLE": "syscall", - "syscall.Madvise": "syscall", - "syscall.MapViewOfFile": "syscall", - "syscall.MaxTokenInfoClass": "syscall", - "syscall.Mclpool": "syscall", - "syscall.MibIfRow": "syscall", - "syscall.Mkdir": "syscall", - "syscall.Mkdirat": "syscall", - "syscall.Mkfifo": "syscall", - "syscall.Mknod": "syscall", - "syscall.Mknodat": "syscall", - "syscall.Mlock": "syscall", - "syscall.Mlockall": "syscall", - "syscall.Mmap": "syscall", - "syscall.Mount": "syscall", - "syscall.MoveFile": "syscall", - "syscall.Mprotect": "syscall", - "syscall.Msghdr": "syscall", - "syscall.Munlock": "syscall", - "syscall.Munlockall": "syscall", - "syscall.Munmap": "syscall", - "syscall.MustLoadDLL": "syscall", - "syscall.NAME_MAX": "syscall", - "syscall.NETLINK_ADD_MEMBERSHIP": "syscall", - "syscall.NETLINK_AUDIT": "syscall", - "syscall.NETLINK_BROADCAST_ERROR": "syscall", - "syscall.NETLINK_CONNECTOR": "syscall", - "syscall.NETLINK_DNRTMSG": "syscall", - "syscall.NETLINK_DROP_MEMBERSHIP": "syscall", - "syscall.NETLINK_ECRYPTFS": "syscall", - "syscall.NETLINK_FIB_LOOKUP": "syscall", - "syscall.NETLINK_FIREWALL": "syscall", - "syscall.NETLINK_GENERIC": "syscall", - "syscall.NETLINK_INET_DIAG": "syscall", - "syscall.NETLINK_IP6_FW": "syscall", - "syscall.NETLINK_ISCSI": "syscall", - "syscall.NETLINK_KOBJECT_UEVENT": "syscall", - "syscall.NETLINK_NETFILTER": "syscall", - "syscall.NETLINK_NFLOG": "syscall", - "syscall.NETLINK_NO_ENOBUFS": "syscall", - "syscall.NETLINK_PKTINFO": "syscall", - "syscall.NETLINK_RDMA": "syscall", - "syscall.NETLINK_ROUTE": "syscall", - "syscall.NETLINK_SCSITRANSPORT": "syscall", - "syscall.NETLINK_SELINUX": "syscall", - "syscall.NETLINK_UNUSED": "syscall", - "syscall.NETLINK_USERSOCK": "syscall", - "syscall.NETLINK_XFRM": "syscall", - "syscall.NET_RT_DUMP": "syscall", - "syscall.NET_RT_DUMP2": "syscall", - "syscall.NET_RT_FLAGS": "syscall", - "syscall.NET_RT_IFLIST": "syscall", - "syscall.NET_RT_IFLIST2": "syscall", - "syscall.NET_RT_IFLISTL": "syscall", - "syscall.NET_RT_IFMALIST": "syscall", - "syscall.NET_RT_MAXID": "syscall", - "syscall.NET_RT_OIFLIST": "syscall", - "syscall.NET_RT_OOIFLIST": "syscall", - "syscall.NET_RT_STAT": "syscall", - "syscall.NET_RT_STATS": "syscall", - "syscall.NET_RT_TABLE": "syscall", - "syscall.NET_RT_TRASH": "syscall", - "syscall.NLA_ALIGNTO": "syscall", - "syscall.NLA_F_NESTED": "syscall", - "syscall.NLA_F_NET_BYTEORDER": "syscall", - "syscall.NLA_HDRLEN": "syscall", - "syscall.NLMSG_ALIGNTO": "syscall", - "syscall.NLMSG_DONE": "syscall", - "syscall.NLMSG_ERROR": "syscall", - "syscall.NLMSG_HDRLEN": "syscall", - "syscall.NLMSG_MIN_TYPE": "syscall", - "syscall.NLMSG_NOOP": "syscall", - "syscall.NLMSG_OVERRUN": "syscall", - "syscall.NLM_F_ACK": "syscall", - "syscall.NLM_F_APPEND": "syscall", - "syscall.NLM_F_ATOMIC": "syscall", - "syscall.NLM_F_CREATE": "syscall", - "syscall.NLM_F_DUMP": "syscall", - "syscall.NLM_F_ECHO": "syscall", - "syscall.NLM_F_EXCL": "syscall", - "syscall.NLM_F_MATCH": "syscall", - "syscall.NLM_F_MULTI": "syscall", - "syscall.NLM_F_REPLACE": "syscall", - "syscall.NLM_F_REQUEST": "syscall", - "syscall.NLM_F_ROOT": "syscall", - "syscall.NOFLSH": "syscall", - "syscall.NOTE_ABSOLUTE": "syscall", - "syscall.NOTE_ATTRIB": "syscall", - "syscall.NOTE_CHILD": "syscall", - "syscall.NOTE_DELETE": "syscall", - "syscall.NOTE_EOF": "syscall", - "syscall.NOTE_EXEC": "syscall", - "syscall.NOTE_EXIT": "syscall", - "syscall.NOTE_EXITSTATUS": "syscall", - "syscall.NOTE_EXTEND": "syscall", - "syscall.NOTE_FFAND": "syscall", - "syscall.NOTE_FFCOPY": "syscall", - "syscall.NOTE_FFCTRLMASK": "syscall", - "syscall.NOTE_FFLAGSMASK": "syscall", - "syscall.NOTE_FFNOP": "syscall", - "syscall.NOTE_FFOR": "syscall", - "syscall.NOTE_FORK": "syscall", - "syscall.NOTE_LINK": "syscall", - "syscall.NOTE_LOWAT": "syscall", - "syscall.NOTE_NONE": "syscall", - "syscall.NOTE_NSECONDS": "syscall", - "syscall.NOTE_PCTRLMASK": "syscall", - "syscall.NOTE_PDATAMASK": "syscall", - "syscall.NOTE_REAP": "syscall", - "syscall.NOTE_RENAME": "syscall", - "syscall.NOTE_RESOURCEEND": "syscall", - "syscall.NOTE_REVOKE": "syscall", - "syscall.NOTE_SECONDS": "syscall", - "syscall.NOTE_SIGNAL": "syscall", - "syscall.NOTE_TRACK": "syscall", - "syscall.NOTE_TRACKERR": "syscall", - "syscall.NOTE_TRIGGER": "syscall", - "syscall.NOTE_TRUNCATE": "syscall", - "syscall.NOTE_USECONDS": "syscall", - "syscall.NOTE_VM_ERROR": "syscall", - "syscall.NOTE_VM_PRESSURE": "syscall", - "syscall.NOTE_VM_PRESSURE_SUDDEN_TERMINATE": "syscall", - "syscall.NOTE_VM_PRESSURE_TERMINATE": "syscall", - "syscall.NOTE_WRITE": "syscall", - "syscall.NameCanonical": "syscall", - "syscall.NameCanonicalEx": "syscall", - "syscall.NameDisplay": "syscall", - "syscall.NameDnsDomain": "syscall", - "syscall.NameFullyQualifiedDN": "syscall", - "syscall.NameSamCompatible": "syscall", - "syscall.NameServicePrincipal": "syscall", - "syscall.NameUniqueId": "syscall", - "syscall.NameUnknown": "syscall", - "syscall.NameUserPrincipal": "syscall", - "syscall.Nanosleep": "syscall", - "syscall.NetApiBufferFree": "syscall", - "syscall.NetGetJoinInformation": "syscall", - "syscall.NetSetupDomainName": "syscall", - "syscall.NetSetupUnjoined": "syscall", - "syscall.NetSetupUnknownStatus": "syscall", - "syscall.NetSetupWorkgroupName": "syscall", - "syscall.NetUserGetInfo": "syscall", - "syscall.NetlinkMessage": "syscall", - "syscall.NetlinkRIB": "syscall", - "syscall.NetlinkRouteAttr": "syscall", - "syscall.NetlinkRouteRequest": "syscall", - "syscall.NewCallback": "syscall", - "syscall.NewCallbackCDecl": "syscall", - "syscall.NewLazyDLL": "syscall", - "syscall.NlAttr": "syscall", - "syscall.NlMsgerr": "syscall", - "syscall.NlMsghdr": "syscall", - "syscall.NsecToFiletime": "syscall", - "syscall.NsecToTimespec": "syscall", - "syscall.NsecToTimeval": "syscall", - "syscall.Ntohs": "syscall", - "syscall.OCRNL": "syscall", - "syscall.OFDEL": "syscall", - "syscall.OFILL": "syscall", - "syscall.OFIOGETBMAP": "syscall", - "syscall.OID_PKIX_KP_SERVER_AUTH": "syscall", - "syscall.OID_SERVER_GATED_CRYPTO": "syscall", - "syscall.OID_SGC_NETSCAPE": "syscall", - "syscall.OLCUC": "syscall", - "syscall.ONLCR": "syscall", - "syscall.ONLRET": "syscall", - "syscall.ONOCR": "syscall", - "syscall.ONOEOT": "syscall", - "syscall.OPEN_ALWAYS": "syscall", - "syscall.OPEN_EXISTING": "syscall", - "syscall.OPOST": "syscall", - "syscall.O_ACCMODE": "syscall", - "syscall.O_ALERT": "syscall", - "syscall.O_ALT_IO": "syscall", - "syscall.O_APPEND": "syscall", - "syscall.O_ASYNC": "syscall", - "syscall.O_CLOEXEC": "syscall", - "syscall.O_CREAT": "syscall", - "syscall.O_DIRECT": "syscall", - "syscall.O_DIRECTORY": "syscall", - "syscall.O_DSYNC": "syscall", - "syscall.O_EVTONLY": "syscall", - "syscall.O_EXCL": "syscall", - "syscall.O_EXEC": "syscall", - "syscall.O_EXLOCK": "syscall", - "syscall.O_FSYNC": "syscall", - "syscall.O_LARGEFILE": "syscall", - "syscall.O_NDELAY": "syscall", - "syscall.O_NOATIME": "syscall", - "syscall.O_NOCTTY": "syscall", - "syscall.O_NOFOLLOW": "syscall", - "syscall.O_NONBLOCK": "syscall", - "syscall.O_NOSIGPIPE": "syscall", - "syscall.O_POPUP": "syscall", - "syscall.O_RDONLY": "syscall", - "syscall.O_RDWR": "syscall", - "syscall.O_RSYNC": "syscall", - "syscall.O_SHLOCK": "syscall", - "syscall.O_SYMLINK": "syscall", - "syscall.O_SYNC": "syscall", - "syscall.O_TRUNC": "syscall", - "syscall.O_TTY_INIT": "syscall", - "syscall.O_WRONLY": "syscall", - "syscall.Open": "syscall", - "syscall.OpenCurrentProcessToken": "syscall", - "syscall.OpenProcess": "syscall", - "syscall.OpenProcessToken": "syscall", - "syscall.Openat": "syscall", - "syscall.Overlapped": "syscall", - "syscall.PACKET_ADD_MEMBERSHIP": "syscall", - "syscall.PACKET_BROADCAST": "syscall", - "syscall.PACKET_DROP_MEMBERSHIP": "syscall", - "syscall.PACKET_FASTROUTE": "syscall", - "syscall.PACKET_HOST": "syscall", - "syscall.PACKET_LOOPBACK": "syscall", - "syscall.PACKET_MR_ALLMULTI": "syscall", - "syscall.PACKET_MR_MULTICAST": "syscall", - "syscall.PACKET_MR_PROMISC": "syscall", - "syscall.PACKET_MULTICAST": "syscall", - "syscall.PACKET_OTHERHOST": "syscall", - "syscall.PACKET_OUTGOING": "syscall", - "syscall.PACKET_RECV_OUTPUT": "syscall", - "syscall.PACKET_RX_RING": "syscall", - "syscall.PACKET_STATISTICS": "syscall", - "syscall.PAGE_EXECUTE_READ": "syscall", - "syscall.PAGE_EXECUTE_READWRITE": "syscall", - "syscall.PAGE_EXECUTE_WRITECOPY": "syscall", - "syscall.PAGE_READONLY": "syscall", - "syscall.PAGE_READWRITE": "syscall", - "syscall.PAGE_WRITECOPY": "syscall", - "syscall.PARENB": "syscall", - "syscall.PARMRK": "syscall", - "syscall.PARODD": "syscall", - "syscall.PENDIN": "syscall", - "syscall.PFL_HIDDEN": "syscall", - "syscall.PFL_MATCHES_PROTOCOL_ZERO": "syscall", - "syscall.PFL_MULTIPLE_PROTO_ENTRIES": "syscall", - "syscall.PFL_NETWORKDIRECT_PROVIDER": "syscall", - "syscall.PFL_RECOMMENDED_PROTO_ENTRY": "syscall", - "syscall.PF_FLUSH": "syscall", - "syscall.PKCS_7_ASN_ENCODING": "syscall", - "syscall.PMC5_PIPELINE_FLUSH": "syscall", - "syscall.PRIO_PGRP": "syscall", - "syscall.PRIO_PROCESS": "syscall", - "syscall.PRIO_USER": "syscall", - "syscall.PRI_IOFLUSH": "syscall", - "syscall.PROCESS_QUERY_INFORMATION": "syscall", - "syscall.PROCESS_TERMINATE": "syscall", - "syscall.PROT_EXEC": "syscall", - "syscall.PROT_GROWSDOWN": "syscall", - "syscall.PROT_GROWSUP": "syscall", - "syscall.PROT_NONE": "syscall", - "syscall.PROT_READ": "syscall", - "syscall.PROT_WRITE": "syscall", - "syscall.PROV_DH_SCHANNEL": "syscall", - "syscall.PROV_DSS": "syscall", - "syscall.PROV_DSS_DH": "syscall", - "syscall.PROV_EC_ECDSA_FULL": "syscall", - "syscall.PROV_EC_ECDSA_SIG": "syscall", - "syscall.PROV_EC_ECNRA_FULL": "syscall", - "syscall.PROV_EC_ECNRA_SIG": "syscall", - "syscall.PROV_FORTEZZA": "syscall", - "syscall.PROV_INTEL_SEC": "syscall", - "syscall.PROV_MS_EXCHANGE": "syscall", - "syscall.PROV_REPLACE_OWF": "syscall", - "syscall.PROV_RNG": "syscall", - "syscall.PROV_RSA_AES": "syscall", - "syscall.PROV_RSA_FULL": "syscall", - "syscall.PROV_RSA_SCHANNEL": "syscall", - "syscall.PROV_RSA_SIG": "syscall", - "syscall.PROV_SPYRUS_LYNKS": "syscall", - "syscall.PROV_SSL": "syscall", - "syscall.PR_CAPBSET_DROP": "syscall", - "syscall.PR_CAPBSET_READ": "syscall", - "syscall.PR_CLEAR_SECCOMP_FILTER": "syscall", - "syscall.PR_ENDIAN_BIG": "syscall", - "syscall.PR_ENDIAN_LITTLE": "syscall", - "syscall.PR_ENDIAN_PPC_LITTLE": "syscall", - "syscall.PR_FPEMU_NOPRINT": "syscall", - "syscall.PR_FPEMU_SIGFPE": "syscall", - "syscall.PR_FP_EXC_ASYNC": "syscall", - "syscall.PR_FP_EXC_DISABLED": "syscall", - "syscall.PR_FP_EXC_DIV": "syscall", - "syscall.PR_FP_EXC_INV": "syscall", - "syscall.PR_FP_EXC_NONRECOV": "syscall", - "syscall.PR_FP_EXC_OVF": "syscall", - "syscall.PR_FP_EXC_PRECISE": "syscall", - "syscall.PR_FP_EXC_RES": "syscall", - "syscall.PR_FP_EXC_SW_ENABLE": "syscall", - "syscall.PR_FP_EXC_UND": "syscall", - "syscall.PR_GET_DUMPABLE": "syscall", - "syscall.PR_GET_ENDIAN": "syscall", - "syscall.PR_GET_FPEMU": "syscall", - "syscall.PR_GET_FPEXC": "syscall", - "syscall.PR_GET_KEEPCAPS": "syscall", - "syscall.PR_GET_NAME": "syscall", - "syscall.PR_GET_PDEATHSIG": "syscall", - "syscall.PR_GET_SECCOMP": "syscall", - "syscall.PR_GET_SECCOMP_FILTER": "syscall", - "syscall.PR_GET_SECUREBITS": "syscall", - "syscall.PR_GET_TIMERSLACK": "syscall", - "syscall.PR_GET_TIMING": "syscall", - "syscall.PR_GET_TSC": "syscall", - "syscall.PR_GET_UNALIGN": "syscall", - "syscall.PR_MCE_KILL": "syscall", - "syscall.PR_MCE_KILL_CLEAR": "syscall", - "syscall.PR_MCE_KILL_DEFAULT": "syscall", - "syscall.PR_MCE_KILL_EARLY": "syscall", - "syscall.PR_MCE_KILL_GET": "syscall", - "syscall.PR_MCE_KILL_LATE": "syscall", - "syscall.PR_MCE_KILL_SET": "syscall", - "syscall.PR_SECCOMP_FILTER_EVENT": "syscall", - "syscall.PR_SECCOMP_FILTER_SYSCALL": "syscall", - "syscall.PR_SET_DUMPABLE": "syscall", - "syscall.PR_SET_ENDIAN": "syscall", - "syscall.PR_SET_FPEMU": "syscall", - "syscall.PR_SET_FPEXC": "syscall", - "syscall.PR_SET_KEEPCAPS": "syscall", - "syscall.PR_SET_NAME": "syscall", - "syscall.PR_SET_PDEATHSIG": "syscall", - "syscall.PR_SET_PTRACER": "syscall", - "syscall.PR_SET_SECCOMP": "syscall", - "syscall.PR_SET_SECCOMP_FILTER": "syscall", - "syscall.PR_SET_SECUREBITS": "syscall", - "syscall.PR_SET_TIMERSLACK": "syscall", - "syscall.PR_SET_TIMING": "syscall", - "syscall.PR_SET_TSC": "syscall", - "syscall.PR_SET_UNALIGN": "syscall", - "syscall.PR_TASK_PERF_EVENTS_DISABLE": "syscall", - "syscall.PR_TASK_PERF_EVENTS_ENABLE": "syscall", - "syscall.PR_TIMING_STATISTICAL": "syscall", - "syscall.PR_TIMING_TIMESTAMP": "syscall", - "syscall.PR_TSC_ENABLE": "syscall", - "syscall.PR_TSC_SIGSEGV": "syscall", - "syscall.PR_UNALIGN_NOPRINT": "syscall", - "syscall.PR_UNALIGN_SIGBUS": "syscall", - "syscall.PTRACE_ARCH_PRCTL": "syscall", - "syscall.PTRACE_ATTACH": "syscall", - "syscall.PTRACE_CONT": "syscall", - "syscall.PTRACE_DETACH": "syscall", - "syscall.PTRACE_EVENT_CLONE": "syscall", - "syscall.PTRACE_EVENT_EXEC": "syscall", - "syscall.PTRACE_EVENT_EXIT": "syscall", - "syscall.PTRACE_EVENT_FORK": "syscall", - "syscall.PTRACE_EVENT_VFORK": "syscall", - "syscall.PTRACE_EVENT_VFORK_DONE": "syscall", - "syscall.PTRACE_GETCRUNCHREGS": "syscall", - "syscall.PTRACE_GETEVENTMSG": "syscall", - "syscall.PTRACE_GETFPREGS": "syscall", - "syscall.PTRACE_GETFPXREGS": "syscall", - "syscall.PTRACE_GETHBPREGS": "syscall", - "syscall.PTRACE_GETREGS": "syscall", - "syscall.PTRACE_GETREGSET": "syscall", - "syscall.PTRACE_GETSIGINFO": "syscall", - "syscall.PTRACE_GETVFPREGS": "syscall", - "syscall.PTRACE_GETWMMXREGS": "syscall", - "syscall.PTRACE_GET_THREAD_AREA": "syscall", - "syscall.PTRACE_KILL": "syscall", - "syscall.PTRACE_OLDSETOPTIONS": "syscall", - "syscall.PTRACE_O_MASK": "syscall", - "syscall.PTRACE_O_TRACECLONE": "syscall", - "syscall.PTRACE_O_TRACEEXEC": "syscall", - "syscall.PTRACE_O_TRACEEXIT": "syscall", - "syscall.PTRACE_O_TRACEFORK": "syscall", - "syscall.PTRACE_O_TRACESYSGOOD": "syscall", - "syscall.PTRACE_O_TRACEVFORK": "syscall", - "syscall.PTRACE_O_TRACEVFORKDONE": "syscall", - "syscall.PTRACE_PEEKDATA": "syscall", - "syscall.PTRACE_PEEKTEXT": "syscall", - "syscall.PTRACE_PEEKUSR": "syscall", - "syscall.PTRACE_POKEDATA": "syscall", - "syscall.PTRACE_POKETEXT": "syscall", - "syscall.PTRACE_POKEUSR": "syscall", - "syscall.PTRACE_SETCRUNCHREGS": "syscall", - "syscall.PTRACE_SETFPREGS": "syscall", - "syscall.PTRACE_SETFPXREGS": "syscall", - "syscall.PTRACE_SETHBPREGS": "syscall", - "syscall.PTRACE_SETOPTIONS": "syscall", - "syscall.PTRACE_SETREGS": "syscall", - "syscall.PTRACE_SETREGSET": "syscall", - "syscall.PTRACE_SETSIGINFO": "syscall", - "syscall.PTRACE_SETVFPREGS": "syscall", - "syscall.PTRACE_SETWMMXREGS": "syscall", - "syscall.PTRACE_SET_SYSCALL": "syscall", - "syscall.PTRACE_SET_THREAD_AREA": "syscall", - "syscall.PTRACE_SINGLEBLOCK": "syscall", - "syscall.PTRACE_SINGLESTEP": "syscall", - "syscall.PTRACE_SYSCALL": "syscall", - "syscall.PTRACE_SYSEMU": "syscall", - "syscall.PTRACE_SYSEMU_SINGLESTEP": "syscall", - "syscall.PTRACE_TRACEME": "syscall", - "syscall.PT_ATTACH": "syscall", - "syscall.PT_ATTACHEXC": "syscall", - "syscall.PT_CONTINUE": "syscall", - "syscall.PT_DATA_ADDR": "syscall", - "syscall.PT_DENY_ATTACH": "syscall", - "syscall.PT_DETACH": "syscall", - "syscall.PT_FIRSTMACH": "syscall", - "syscall.PT_FORCEQUOTA": "syscall", - "syscall.PT_KILL": "syscall", - "syscall.PT_MASK": "syscall", - "syscall.PT_READ_D": "syscall", - "syscall.PT_READ_I": "syscall", - "syscall.PT_READ_U": "syscall", - "syscall.PT_SIGEXC": "syscall", - "syscall.PT_STEP": "syscall", - "syscall.PT_TEXT_ADDR": "syscall", - "syscall.PT_TEXT_END_ADDR": "syscall", - "syscall.PT_THUPDATE": "syscall", - "syscall.PT_TRACE_ME": "syscall", - "syscall.PT_WRITE_D": "syscall", - "syscall.PT_WRITE_I": "syscall", - "syscall.PT_WRITE_U": "syscall", - "syscall.ParseDirent": "syscall", - "syscall.ParseNetlinkMessage": "syscall", - "syscall.ParseNetlinkRouteAttr": "syscall", - "syscall.ParseRoutingMessage": "syscall", - "syscall.ParseRoutingSockaddr": "syscall", - "syscall.ParseSocketControlMessage": "syscall", - "syscall.ParseUnixCredentials": "syscall", - "syscall.ParseUnixRights": "syscall", - "syscall.PathMax": "syscall", - "syscall.Pathconf": "syscall", - "syscall.Pause": "syscall", - "syscall.Pipe": "syscall", - "syscall.Pipe2": "syscall", - "syscall.PivotRoot": "syscall", - "syscall.PostQueuedCompletionStatus": "syscall", - "syscall.Pread": "syscall", - "syscall.Proc": "syscall", - "syscall.ProcAttr": "syscall", - "syscall.Process32First": "syscall", - "syscall.Process32Next": "syscall", - "syscall.ProcessEntry32": "syscall", - "syscall.ProcessInformation": "syscall", - "syscall.Protoent": "syscall", - "syscall.PtraceAttach": "syscall", - "syscall.PtraceCont": "syscall", - "syscall.PtraceDetach": "syscall", - "syscall.PtraceGetEventMsg": "syscall", - "syscall.PtraceGetRegs": "syscall", - "syscall.PtracePeekData": "syscall", - "syscall.PtracePeekText": "syscall", - "syscall.PtracePokeData": "syscall", - "syscall.PtracePokeText": "syscall", - "syscall.PtraceRegs": "syscall", - "syscall.PtraceSetOptions": "syscall", - "syscall.PtraceSetRegs": "syscall", - "syscall.PtraceSingleStep": "syscall", - "syscall.PtraceSyscall": "syscall", - "syscall.Pwrite": "syscall", - "syscall.REG_BINARY": "syscall", - "syscall.REG_DWORD": "syscall", - "syscall.REG_DWORD_BIG_ENDIAN": "syscall", - "syscall.REG_DWORD_LITTLE_ENDIAN": "syscall", - "syscall.REG_EXPAND_SZ": "syscall", - "syscall.REG_FULL_RESOURCE_DESCRIPTOR": "syscall", - "syscall.REG_LINK": "syscall", - "syscall.REG_MULTI_SZ": "syscall", - "syscall.REG_NONE": "syscall", - "syscall.REG_QWORD": "syscall", - "syscall.REG_QWORD_LITTLE_ENDIAN": "syscall", - "syscall.REG_RESOURCE_LIST": "syscall", - "syscall.REG_RESOURCE_REQUIREMENTS_LIST": "syscall", - "syscall.REG_SZ": "syscall", - "syscall.RLIMIT_AS": "syscall", - "syscall.RLIMIT_CORE": "syscall", - "syscall.RLIMIT_CPU": "syscall", - "syscall.RLIMIT_DATA": "syscall", - "syscall.RLIMIT_FSIZE": "syscall", - "syscall.RLIMIT_NOFILE": "syscall", - "syscall.RLIMIT_STACK": "syscall", - "syscall.RLIM_INFINITY": "syscall", - "syscall.RTAX_ADVMSS": "syscall", - "syscall.RTAX_AUTHOR": "syscall", - "syscall.RTAX_BRD": "syscall", - "syscall.RTAX_CWND": "syscall", - "syscall.RTAX_DST": "syscall", - "syscall.RTAX_FEATURES": "syscall", - "syscall.RTAX_FEATURE_ALLFRAG": "syscall", - "syscall.RTAX_FEATURE_ECN": "syscall", - "syscall.RTAX_FEATURE_SACK": "syscall", - "syscall.RTAX_FEATURE_TIMESTAMP": "syscall", - "syscall.RTAX_GATEWAY": "syscall", - "syscall.RTAX_GENMASK": "syscall", - "syscall.RTAX_HOPLIMIT": "syscall", - "syscall.RTAX_IFA": "syscall", - "syscall.RTAX_IFP": "syscall", - "syscall.RTAX_INITCWND": "syscall", - "syscall.RTAX_INITRWND": "syscall", - "syscall.RTAX_LABEL": "syscall", - "syscall.RTAX_LOCK": "syscall", - "syscall.RTAX_MAX": "syscall", - "syscall.RTAX_MTU": "syscall", - "syscall.RTAX_NETMASK": "syscall", - "syscall.RTAX_REORDERING": "syscall", - "syscall.RTAX_RTO_MIN": "syscall", - "syscall.RTAX_RTT": "syscall", - "syscall.RTAX_RTTVAR": "syscall", - "syscall.RTAX_SRC": "syscall", - "syscall.RTAX_SRCMASK": "syscall", - "syscall.RTAX_SSTHRESH": "syscall", - "syscall.RTAX_TAG": "syscall", - "syscall.RTAX_UNSPEC": "syscall", - "syscall.RTAX_WINDOW": "syscall", - "syscall.RTA_ALIGNTO": "syscall", - "syscall.RTA_AUTHOR": "syscall", - "syscall.RTA_BRD": "syscall", - "syscall.RTA_CACHEINFO": "syscall", - "syscall.RTA_DST": "syscall", - "syscall.RTA_FLOW": "syscall", - "syscall.RTA_GATEWAY": "syscall", - "syscall.RTA_GENMASK": "syscall", - "syscall.RTA_IFA": "syscall", - "syscall.RTA_IFP": "syscall", - "syscall.RTA_IIF": "syscall", - "syscall.RTA_LABEL": "syscall", - "syscall.RTA_MAX": "syscall", - "syscall.RTA_METRICS": "syscall", - "syscall.RTA_MULTIPATH": "syscall", - "syscall.RTA_NETMASK": "syscall", - "syscall.RTA_OIF": "syscall", - "syscall.RTA_PREFSRC": "syscall", - "syscall.RTA_PRIORITY": "syscall", - "syscall.RTA_SRC": "syscall", - "syscall.RTA_SRCMASK": "syscall", - "syscall.RTA_TABLE": "syscall", - "syscall.RTA_TAG": "syscall", - "syscall.RTA_UNSPEC": "syscall", - "syscall.RTCF_DIRECTSRC": "syscall", - "syscall.RTCF_DOREDIRECT": "syscall", - "syscall.RTCF_LOG": "syscall", - "syscall.RTCF_MASQ": "syscall", - "syscall.RTCF_NAT": "syscall", - "syscall.RTCF_VALVE": "syscall", - "syscall.RTF_ADDRCLASSMASK": "syscall", - "syscall.RTF_ADDRCONF": "syscall", - "syscall.RTF_ALLONLINK": "syscall", - "syscall.RTF_ANNOUNCE": "syscall", - "syscall.RTF_BLACKHOLE": "syscall", - "syscall.RTF_BROADCAST": "syscall", - "syscall.RTF_CACHE": "syscall", - "syscall.RTF_CLONED": "syscall", - "syscall.RTF_CLONING": "syscall", - "syscall.RTF_CONDEMNED": "syscall", - "syscall.RTF_DEFAULT": "syscall", - "syscall.RTF_DELCLONE": "syscall", - "syscall.RTF_DONE": "syscall", - "syscall.RTF_DYNAMIC": "syscall", - "syscall.RTF_FLOW": "syscall", - "syscall.RTF_FMASK": "syscall", - "syscall.RTF_GATEWAY": "syscall", - "syscall.RTF_GWFLAG_COMPAT": "syscall", - "syscall.RTF_HOST": "syscall", - "syscall.RTF_IFREF": "syscall", - "syscall.RTF_IFSCOPE": "syscall", - "syscall.RTF_INTERFACE": "syscall", - "syscall.RTF_IRTT": "syscall", - "syscall.RTF_LINKRT": "syscall", - "syscall.RTF_LLDATA": "syscall", - "syscall.RTF_LLINFO": "syscall", - "syscall.RTF_LOCAL": "syscall", - "syscall.RTF_MASK": "syscall", - "syscall.RTF_MODIFIED": "syscall", - "syscall.RTF_MPATH": "syscall", - "syscall.RTF_MPLS": "syscall", - "syscall.RTF_MSS": "syscall", - "syscall.RTF_MTU": "syscall", - "syscall.RTF_MULTICAST": "syscall", - "syscall.RTF_NAT": "syscall", - "syscall.RTF_NOFORWARD": "syscall", - "syscall.RTF_NONEXTHOP": "syscall", - "syscall.RTF_NOPMTUDISC": "syscall", - "syscall.RTF_PERMANENT_ARP": "syscall", - "syscall.RTF_PINNED": "syscall", - "syscall.RTF_POLICY": "syscall", - "syscall.RTF_PRCLONING": "syscall", - "syscall.RTF_PROTO1": "syscall", - "syscall.RTF_PROTO2": "syscall", - "syscall.RTF_PROTO3": "syscall", - "syscall.RTF_REINSTATE": "syscall", - "syscall.RTF_REJECT": "syscall", - "syscall.RTF_RNH_LOCKED": "syscall", - "syscall.RTF_SOURCE": "syscall", - "syscall.RTF_SRC": "syscall", - "syscall.RTF_STATIC": "syscall", - "syscall.RTF_STICKY": "syscall", - "syscall.RTF_THROW": "syscall", - "syscall.RTF_TUNNEL": "syscall", - "syscall.RTF_UP": "syscall", - "syscall.RTF_USETRAILERS": "syscall", - "syscall.RTF_WASCLONED": "syscall", - "syscall.RTF_WINDOW": "syscall", - "syscall.RTF_XRESOLVE": "syscall", - "syscall.RTM_ADD": "syscall", - "syscall.RTM_BASE": "syscall", - "syscall.RTM_CHANGE": "syscall", - "syscall.RTM_CHGADDR": "syscall", - "syscall.RTM_DELACTION": "syscall", - "syscall.RTM_DELADDR": "syscall", - "syscall.RTM_DELADDRLABEL": "syscall", - "syscall.RTM_DELETE": "syscall", - "syscall.RTM_DELLINK": "syscall", - "syscall.RTM_DELMADDR": "syscall", - "syscall.RTM_DELNEIGH": "syscall", - "syscall.RTM_DELQDISC": "syscall", - "syscall.RTM_DELROUTE": "syscall", - "syscall.RTM_DELRULE": "syscall", - "syscall.RTM_DELTCLASS": "syscall", - "syscall.RTM_DELTFILTER": "syscall", - "syscall.RTM_DESYNC": "syscall", - "syscall.RTM_F_CLONED": "syscall", - "syscall.RTM_F_EQUALIZE": "syscall", - "syscall.RTM_F_NOTIFY": "syscall", - "syscall.RTM_F_PREFIX": "syscall", - "syscall.RTM_GET": "syscall", - "syscall.RTM_GET2": "syscall", - "syscall.RTM_GETACTION": "syscall", - "syscall.RTM_GETADDR": "syscall", - "syscall.RTM_GETADDRLABEL": "syscall", - "syscall.RTM_GETANYCAST": "syscall", - "syscall.RTM_GETDCB": "syscall", - "syscall.RTM_GETLINK": "syscall", - "syscall.RTM_GETMULTICAST": "syscall", - "syscall.RTM_GETNEIGH": "syscall", - "syscall.RTM_GETNEIGHTBL": "syscall", - "syscall.RTM_GETQDISC": "syscall", - "syscall.RTM_GETROUTE": "syscall", - "syscall.RTM_GETRULE": "syscall", - "syscall.RTM_GETTCLASS": "syscall", - "syscall.RTM_GETTFILTER": "syscall", - "syscall.RTM_IEEE80211": "syscall", - "syscall.RTM_IFANNOUNCE": "syscall", - "syscall.RTM_IFINFO": "syscall", - "syscall.RTM_IFINFO2": "syscall", - "syscall.RTM_LLINFO_UPD": "syscall", - "syscall.RTM_LOCK": "syscall", - "syscall.RTM_LOSING": "syscall", - "syscall.RTM_MAX": "syscall", - "syscall.RTM_MAXSIZE": "syscall", - "syscall.RTM_MISS": "syscall", - "syscall.RTM_NEWACTION": "syscall", - "syscall.RTM_NEWADDR": "syscall", - "syscall.RTM_NEWADDRLABEL": "syscall", - "syscall.RTM_NEWLINK": "syscall", - "syscall.RTM_NEWMADDR": "syscall", - "syscall.RTM_NEWMADDR2": "syscall", - "syscall.RTM_NEWNDUSEROPT": "syscall", - "syscall.RTM_NEWNEIGH": "syscall", - "syscall.RTM_NEWNEIGHTBL": "syscall", - "syscall.RTM_NEWPREFIX": "syscall", - "syscall.RTM_NEWQDISC": "syscall", - "syscall.RTM_NEWROUTE": "syscall", - "syscall.RTM_NEWRULE": "syscall", - "syscall.RTM_NEWTCLASS": "syscall", - "syscall.RTM_NEWTFILTER": "syscall", - "syscall.RTM_NR_FAMILIES": "syscall", - "syscall.RTM_NR_MSGTYPES": "syscall", - "syscall.RTM_OIFINFO": "syscall", - "syscall.RTM_OLDADD": "syscall", - "syscall.RTM_OLDDEL": "syscall", - "syscall.RTM_OOIFINFO": "syscall", - "syscall.RTM_REDIRECT": "syscall", - "syscall.RTM_RESOLVE": "syscall", - "syscall.RTM_RTTUNIT": "syscall", - "syscall.RTM_SETDCB": "syscall", - "syscall.RTM_SETGATE": "syscall", - "syscall.RTM_SETLINK": "syscall", - "syscall.RTM_SETNEIGHTBL": "syscall", - "syscall.RTM_VERSION": "syscall", - "syscall.RTNH_ALIGNTO": "syscall", - "syscall.RTNH_F_DEAD": "syscall", - "syscall.RTNH_F_ONLINK": "syscall", - "syscall.RTNH_F_PERVASIVE": "syscall", - "syscall.RTNLGRP_IPV4_IFADDR": "syscall", - "syscall.RTNLGRP_IPV4_MROUTE": "syscall", - "syscall.RTNLGRP_IPV4_ROUTE": "syscall", - "syscall.RTNLGRP_IPV4_RULE": "syscall", - "syscall.RTNLGRP_IPV6_IFADDR": "syscall", - "syscall.RTNLGRP_IPV6_IFINFO": "syscall", - "syscall.RTNLGRP_IPV6_MROUTE": "syscall", - "syscall.RTNLGRP_IPV6_PREFIX": "syscall", - "syscall.RTNLGRP_IPV6_ROUTE": "syscall", - "syscall.RTNLGRP_IPV6_RULE": "syscall", - "syscall.RTNLGRP_LINK": "syscall", - "syscall.RTNLGRP_ND_USEROPT": "syscall", - "syscall.RTNLGRP_NEIGH": "syscall", - "syscall.RTNLGRP_NONE": "syscall", - "syscall.RTNLGRP_NOTIFY": "syscall", - "syscall.RTNLGRP_TC": "syscall", - "syscall.RTN_ANYCAST": "syscall", - "syscall.RTN_BLACKHOLE": "syscall", - "syscall.RTN_BROADCAST": "syscall", - "syscall.RTN_LOCAL": "syscall", - "syscall.RTN_MAX": "syscall", - "syscall.RTN_MULTICAST": "syscall", - "syscall.RTN_NAT": "syscall", - "syscall.RTN_PROHIBIT": "syscall", - "syscall.RTN_THROW": "syscall", - "syscall.RTN_UNICAST": "syscall", - "syscall.RTN_UNREACHABLE": "syscall", - "syscall.RTN_UNSPEC": "syscall", - "syscall.RTN_XRESOLVE": "syscall", - "syscall.RTPROT_BIRD": "syscall", - "syscall.RTPROT_BOOT": "syscall", - "syscall.RTPROT_DHCP": "syscall", - "syscall.RTPROT_DNROUTED": "syscall", - "syscall.RTPROT_GATED": "syscall", - "syscall.RTPROT_KERNEL": "syscall", - "syscall.RTPROT_MRT": "syscall", - "syscall.RTPROT_NTK": "syscall", - "syscall.RTPROT_RA": "syscall", - "syscall.RTPROT_REDIRECT": "syscall", - "syscall.RTPROT_STATIC": "syscall", - "syscall.RTPROT_UNSPEC": "syscall", - "syscall.RTPROT_XORP": "syscall", - "syscall.RTPROT_ZEBRA": "syscall", - "syscall.RTV_EXPIRE": "syscall", - "syscall.RTV_HOPCOUNT": "syscall", - "syscall.RTV_MTU": "syscall", - "syscall.RTV_RPIPE": "syscall", - "syscall.RTV_RTT": "syscall", - "syscall.RTV_RTTVAR": "syscall", - "syscall.RTV_SPIPE": "syscall", - "syscall.RTV_SSTHRESH": "syscall", - "syscall.RTV_WEIGHT": "syscall", - "syscall.RT_CACHING_CONTEXT": "syscall", - "syscall.RT_CLASS_DEFAULT": "syscall", - "syscall.RT_CLASS_LOCAL": "syscall", - "syscall.RT_CLASS_MAIN": "syscall", - "syscall.RT_CLASS_MAX": "syscall", - "syscall.RT_CLASS_UNSPEC": "syscall", - "syscall.RT_DEFAULT_FIB": "syscall", - "syscall.RT_NORTREF": "syscall", - "syscall.RT_SCOPE_HOST": "syscall", - "syscall.RT_SCOPE_LINK": "syscall", - "syscall.RT_SCOPE_NOWHERE": "syscall", - "syscall.RT_SCOPE_SITE": "syscall", - "syscall.RT_SCOPE_UNIVERSE": "syscall", - "syscall.RT_TABLEID_MAX": "syscall", - "syscall.RT_TABLE_COMPAT": "syscall", - "syscall.RT_TABLE_DEFAULT": "syscall", - "syscall.RT_TABLE_LOCAL": "syscall", - "syscall.RT_TABLE_MAIN": "syscall", - "syscall.RT_TABLE_MAX": "syscall", - "syscall.RT_TABLE_UNSPEC": "syscall", - "syscall.RUSAGE_CHILDREN": "syscall", - "syscall.RUSAGE_SELF": "syscall", - "syscall.RUSAGE_THREAD": "syscall", - "syscall.Radvisory_t": "syscall", - "syscall.RawSockaddr": "syscall", - "syscall.RawSockaddrAny": "syscall", - "syscall.RawSockaddrDatalink": "syscall", - "syscall.RawSockaddrInet4": "syscall", - "syscall.RawSockaddrInet6": "syscall", - "syscall.RawSockaddrLinklayer": "syscall", - "syscall.RawSockaddrNetlink": "syscall", - "syscall.RawSockaddrUnix": "syscall", - "syscall.RawSyscall": "syscall", - "syscall.RawSyscall6": "syscall", - "syscall.Read": "syscall", - "syscall.ReadConsole": "syscall", - "syscall.ReadDirectoryChanges": "syscall", - "syscall.ReadDirent": "syscall", - "syscall.ReadFile": "syscall", - "syscall.Readlink": "syscall", - "syscall.Reboot": "syscall", - "syscall.Recvfrom": "syscall", - "syscall.Recvmsg": "syscall", - "syscall.RegCloseKey": "syscall", - "syscall.RegEnumKeyEx": "syscall", - "syscall.RegOpenKeyEx": "syscall", - "syscall.RegQueryInfoKey": "syscall", - "syscall.RegQueryValueEx": "syscall", - "syscall.RemoveDirectory": "syscall", - "syscall.Removexattr": "syscall", - "syscall.Rename": "syscall", - "syscall.Renameat": "syscall", - "syscall.Revoke": "syscall", - "syscall.Rlimit": "syscall", - "syscall.Rmdir": "syscall", - "syscall.RouteMessage": "syscall", - "syscall.RouteRIB": "syscall", - "syscall.RtAttr": "syscall", - "syscall.RtGenmsg": "syscall", - "syscall.RtMetrics": "syscall", - "syscall.RtMsg": "syscall", - "syscall.RtMsghdr": "syscall", - "syscall.RtNexthop": "syscall", - "syscall.Rusage": "syscall", - "syscall.SCM_BINTIME": "syscall", - "syscall.SCM_CREDENTIALS": "syscall", - "syscall.SCM_CREDS": "syscall", - "syscall.SCM_RIGHTS": "syscall", - "syscall.SCM_TIMESTAMP": "syscall", - "syscall.SCM_TIMESTAMPING": "syscall", - "syscall.SCM_TIMESTAMPNS": "syscall", - "syscall.SCM_TIMESTAMP_MONOTONIC": "syscall", - "syscall.SHUT_RD": "syscall", - "syscall.SHUT_RDWR": "syscall", - "syscall.SHUT_WR": "syscall", - "syscall.SID": "syscall", - "syscall.SIDAndAttributes": "syscall", - "syscall.SIGABRT": "syscall", - "syscall.SIGALRM": "syscall", - "syscall.SIGBUS": "syscall", - "syscall.SIGCHLD": "syscall", - "syscall.SIGCLD": "syscall", - "syscall.SIGCONT": "syscall", - "syscall.SIGEMT": "syscall", - "syscall.SIGFPE": "syscall", - "syscall.SIGHUP": "syscall", - "syscall.SIGILL": "syscall", - "syscall.SIGINFO": "syscall", - "syscall.SIGINT": "syscall", - "syscall.SIGIO": "syscall", - "syscall.SIGIOT": "syscall", - "syscall.SIGKILL": "syscall", - "syscall.SIGLIBRT": "syscall", - "syscall.SIGLWP": "syscall", - "syscall.SIGPIPE": "syscall", - "syscall.SIGPOLL": "syscall", - "syscall.SIGPROF": "syscall", - "syscall.SIGPWR": "syscall", - "syscall.SIGQUIT": "syscall", - "syscall.SIGSEGV": "syscall", - "syscall.SIGSTKFLT": "syscall", - "syscall.SIGSTOP": "syscall", - "syscall.SIGSYS": "syscall", - "syscall.SIGTERM": "syscall", - "syscall.SIGTHR": "syscall", - "syscall.SIGTRAP": "syscall", - "syscall.SIGTSTP": "syscall", - "syscall.SIGTTIN": "syscall", - "syscall.SIGTTOU": "syscall", - "syscall.SIGUNUSED": "syscall", - "syscall.SIGURG": "syscall", - "syscall.SIGUSR1": "syscall", - "syscall.SIGUSR2": "syscall", - "syscall.SIGVTALRM": "syscall", - "syscall.SIGWINCH": "syscall", - "syscall.SIGXCPU": "syscall", - "syscall.SIGXFSZ": "syscall", - "syscall.SIOCADDDLCI": "syscall", - "syscall.SIOCADDMULTI": "syscall", - "syscall.SIOCADDRT": "syscall", - "syscall.SIOCAIFADDR": "syscall", - "syscall.SIOCAIFGROUP": "syscall", - "syscall.SIOCALIFADDR": "syscall", - "syscall.SIOCARPIPLL": "syscall", - "syscall.SIOCATMARK": "syscall", - "syscall.SIOCAUTOADDR": "syscall", - "syscall.SIOCAUTONETMASK": "syscall", - "syscall.SIOCBRDGADD": "syscall", - "syscall.SIOCBRDGADDS": "syscall", - "syscall.SIOCBRDGARL": "syscall", - "syscall.SIOCBRDGDADDR": "syscall", - "syscall.SIOCBRDGDEL": "syscall", - "syscall.SIOCBRDGDELS": "syscall", - "syscall.SIOCBRDGFLUSH": "syscall", - "syscall.SIOCBRDGFRL": "syscall", - "syscall.SIOCBRDGGCACHE": "syscall", - "syscall.SIOCBRDGGFD": "syscall", - "syscall.SIOCBRDGGHT": "syscall", - "syscall.SIOCBRDGGIFFLGS": "syscall", - "syscall.SIOCBRDGGMA": "syscall", - "syscall.SIOCBRDGGPARAM": "syscall", - "syscall.SIOCBRDGGPRI": "syscall", - "syscall.SIOCBRDGGRL": "syscall", - "syscall.SIOCBRDGGSIFS": "syscall", - "syscall.SIOCBRDGGTO": "syscall", - "syscall.SIOCBRDGIFS": "syscall", - "syscall.SIOCBRDGRTS": "syscall", - "syscall.SIOCBRDGSADDR": "syscall", - "syscall.SIOCBRDGSCACHE": "syscall", - "syscall.SIOCBRDGSFD": "syscall", - "syscall.SIOCBRDGSHT": "syscall", - "syscall.SIOCBRDGSIFCOST": "syscall", - "syscall.SIOCBRDGSIFFLGS": "syscall", - "syscall.SIOCBRDGSIFPRIO": "syscall", - "syscall.SIOCBRDGSMA": "syscall", - "syscall.SIOCBRDGSPRI": "syscall", - "syscall.SIOCBRDGSPROTO": "syscall", - "syscall.SIOCBRDGSTO": "syscall", - "syscall.SIOCBRDGSTXHC": "syscall", - "syscall.SIOCDARP": "syscall", - "syscall.SIOCDELDLCI": "syscall", - "syscall.SIOCDELMULTI": "syscall", - "syscall.SIOCDELRT": "syscall", - "syscall.SIOCDEVPRIVATE": "syscall", - "syscall.SIOCDIFADDR": "syscall", - "syscall.SIOCDIFGROUP": "syscall", - "syscall.SIOCDIFPHYADDR": "syscall", - "syscall.SIOCDLIFADDR": "syscall", - "syscall.SIOCDRARP": "syscall", - "syscall.SIOCGARP": "syscall", - "syscall.SIOCGDRVSPEC": "syscall", - "syscall.SIOCGETKALIVE": "syscall", - "syscall.SIOCGETLABEL": "syscall", - "syscall.SIOCGETPFLOW": "syscall", - "syscall.SIOCGETPFSYNC": "syscall", - "syscall.SIOCGETSGCNT": "syscall", - "syscall.SIOCGETVIFCNT": "syscall", - "syscall.SIOCGETVLAN": "syscall", - "syscall.SIOCGHIWAT": "syscall", - "syscall.SIOCGIFADDR": "syscall", - "syscall.SIOCGIFADDRPREF": "syscall", - "syscall.SIOCGIFALIAS": "syscall", - "syscall.SIOCGIFALTMTU": "syscall", - "syscall.SIOCGIFASYNCMAP": "syscall", - "syscall.SIOCGIFBOND": "syscall", - "syscall.SIOCGIFBR": "syscall", - "syscall.SIOCGIFBRDADDR": "syscall", - "syscall.SIOCGIFCAP": "syscall", - "syscall.SIOCGIFCONF": "syscall", - "syscall.SIOCGIFCOUNT": "syscall", - "syscall.SIOCGIFDATA": "syscall", - "syscall.SIOCGIFDESCR": "syscall", - "syscall.SIOCGIFDEVMTU": "syscall", - "syscall.SIOCGIFDLT": "syscall", - "syscall.SIOCGIFDSTADDR": "syscall", - "syscall.SIOCGIFENCAP": "syscall", - "syscall.SIOCGIFFIB": "syscall", - "syscall.SIOCGIFFLAGS": "syscall", - "syscall.SIOCGIFGATTR": "syscall", - "syscall.SIOCGIFGENERIC": "syscall", - "syscall.SIOCGIFGMEMB": "syscall", - "syscall.SIOCGIFGROUP": "syscall", - "syscall.SIOCGIFHARDMTU": "syscall", - "syscall.SIOCGIFHWADDR": "syscall", - "syscall.SIOCGIFINDEX": "syscall", - "syscall.SIOCGIFKPI": "syscall", - "syscall.SIOCGIFMAC": "syscall", - "syscall.SIOCGIFMAP": "syscall", - "syscall.SIOCGIFMEDIA": "syscall", - "syscall.SIOCGIFMEM": "syscall", - "syscall.SIOCGIFMETRIC": "syscall", - "syscall.SIOCGIFMTU": "syscall", - "syscall.SIOCGIFNAME": "syscall", - "syscall.SIOCGIFNETMASK": "syscall", - "syscall.SIOCGIFPDSTADDR": "syscall", - "syscall.SIOCGIFPFLAGS": "syscall", - "syscall.SIOCGIFPHYS": "syscall", - "syscall.SIOCGIFPRIORITY": "syscall", - "syscall.SIOCGIFPSRCADDR": "syscall", - "syscall.SIOCGIFRDOMAIN": "syscall", - "syscall.SIOCGIFRTLABEL": "syscall", - "syscall.SIOCGIFSLAVE": "syscall", - "syscall.SIOCGIFSTATUS": "syscall", - "syscall.SIOCGIFTIMESLOT": "syscall", - "syscall.SIOCGIFTXQLEN": "syscall", - "syscall.SIOCGIFVLAN": "syscall", - "syscall.SIOCGIFWAKEFLAGS": "syscall", - "syscall.SIOCGIFXFLAGS": "syscall", - "syscall.SIOCGLIFADDR": "syscall", - "syscall.SIOCGLIFPHYADDR": "syscall", - "syscall.SIOCGLIFPHYRTABLE": "syscall", - "syscall.SIOCGLIFPHYTTL": "syscall", - "syscall.SIOCGLINKSTR": "syscall", - "syscall.SIOCGLOWAT": "syscall", - "syscall.SIOCGPGRP": "syscall", - "syscall.SIOCGPRIVATE_0": "syscall", - "syscall.SIOCGPRIVATE_1": "syscall", - "syscall.SIOCGRARP": "syscall", - "syscall.SIOCGSPPPPARAMS": "syscall", - "syscall.SIOCGSTAMP": "syscall", - "syscall.SIOCGSTAMPNS": "syscall", - "syscall.SIOCGVH": "syscall", - "syscall.SIOCGVNETID": "syscall", - "syscall.SIOCIFCREATE": "syscall", - "syscall.SIOCIFCREATE2": "syscall", - "syscall.SIOCIFDESTROY": "syscall", - "syscall.SIOCIFGCLONERS": "syscall", - "syscall.SIOCINITIFADDR": "syscall", - "syscall.SIOCPROTOPRIVATE": "syscall", - "syscall.SIOCRSLVMULTI": "syscall", - "syscall.SIOCRTMSG": "syscall", - "syscall.SIOCSARP": "syscall", - "syscall.SIOCSDRVSPEC": "syscall", - "syscall.SIOCSETKALIVE": "syscall", - "syscall.SIOCSETLABEL": "syscall", - "syscall.SIOCSETPFLOW": "syscall", - "syscall.SIOCSETPFSYNC": "syscall", - "syscall.SIOCSETVLAN": "syscall", - "syscall.SIOCSHIWAT": "syscall", - "syscall.SIOCSIFADDR": "syscall", - "syscall.SIOCSIFADDRPREF": "syscall", - "syscall.SIOCSIFALTMTU": "syscall", - "syscall.SIOCSIFASYNCMAP": "syscall", - "syscall.SIOCSIFBOND": "syscall", - "syscall.SIOCSIFBR": "syscall", - "syscall.SIOCSIFBRDADDR": "syscall", - "syscall.SIOCSIFCAP": "syscall", - "syscall.SIOCSIFDESCR": "syscall", - "syscall.SIOCSIFDSTADDR": "syscall", - "syscall.SIOCSIFENCAP": "syscall", - "syscall.SIOCSIFFIB": "syscall", - "syscall.SIOCSIFFLAGS": "syscall", - "syscall.SIOCSIFGATTR": "syscall", - "syscall.SIOCSIFGENERIC": "syscall", - "syscall.SIOCSIFHWADDR": "syscall", - "syscall.SIOCSIFHWBROADCAST": "syscall", - "syscall.SIOCSIFKPI": "syscall", - "syscall.SIOCSIFLINK": "syscall", - "syscall.SIOCSIFLLADDR": "syscall", - "syscall.SIOCSIFMAC": "syscall", - "syscall.SIOCSIFMAP": "syscall", - "syscall.SIOCSIFMEDIA": "syscall", - "syscall.SIOCSIFMEM": "syscall", - "syscall.SIOCSIFMETRIC": "syscall", - "syscall.SIOCSIFMTU": "syscall", - "syscall.SIOCSIFNAME": "syscall", - "syscall.SIOCSIFNETMASK": "syscall", - "syscall.SIOCSIFPFLAGS": "syscall", - "syscall.SIOCSIFPHYADDR": "syscall", - "syscall.SIOCSIFPHYS": "syscall", - "syscall.SIOCSIFPRIORITY": "syscall", - "syscall.SIOCSIFRDOMAIN": "syscall", - "syscall.SIOCSIFRTLABEL": "syscall", - "syscall.SIOCSIFRVNET": "syscall", - "syscall.SIOCSIFSLAVE": "syscall", - "syscall.SIOCSIFTIMESLOT": "syscall", - "syscall.SIOCSIFTXQLEN": "syscall", - "syscall.SIOCSIFVLAN": "syscall", - "syscall.SIOCSIFVNET": "syscall", - "syscall.SIOCSIFXFLAGS": "syscall", - "syscall.SIOCSLIFPHYADDR": "syscall", - "syscall.SIOCSLIFPHYRTABLE": "syscall", - "syscall.SIOCSLIFPHYTTL": "syscall", - "syscall.SIOCSLINKSTR": "syscall", - "syscall.SIOCSLOWAT": "syscall", - "syscall.SIOCSPGRP": "syscall", - "syscall.SIOCSRARP": "syscall", - "syscall.SIOCSSPPPPARAMS": "syscall", - "syscall.SIOCSVH": "syscall", - "syscall.SIOCSVNETID": "syscall", - "syscall.SIOCZIFDATA": "syscall", - "syscall.SIO_GET_EXTENSION_FUNCTION_POINTER": "syscall", - "syscall.SIO_GET_INTERFACE_LIST": "syscall", - "syscall.SIO_KEEPALIVE_VALS": "syscall", - "syscall.SIO_UDP_CONNRESET": "syscall", - "syscall.SOCK_CLOEXEC": "syscall", - "syscall.SOCK_DCCP": "syscall", - "syscall.SOCK_DGRAM": "syscall", - "syscall.SOCK_FLAGS_MASK": "syscall", - "syscall.SOCK_MAXADDRLEN": "syscall", - "syscall.SOCK_NONBLOCK": "syscall", - "syscall.SOCK_NOSIGPIPE": "syscall", - "syscall.SOCK_PACKET": "syscall", - "syscall.SOCK_RAW": "syscall", - "syscall.SOCK_RDM": "syscall", - "syscall.SOCK_SEQPACKET": "syscall", - "syscall.SOCK_STREAM": "syscall", - "syscall.SOL_AAL": "syscall", - "syscall.SOL_ATM": "syscall", - "syscall.SOL_DECNET": "syscall", - "syscall.SOL_ICMPV6": "syscall", - "syscall.SOL_IP": "syscall", - "syscall.SOL_IPV6": "syscall", - "syscall.SOL_IRDA": "syscall", - "syscall.SOL_PACKET": "syscall", - "syscall.SOL_RAW": "syscall", - "syscall.SOL_SOCKET": "syscall", - "syscall.SOL_TCP": "syscall", - "syscall.SOL_X25": "syscall", - "syscall.SOMAXCONN": "syscall", - "syscall.SO_ACCEPTCONN": "syscall", - "syscall.SO_ACCEPTFILTER": "syscall", - "syscall.SO_ATTACH_FILTER": "syscall", - "syscall.SO_BINDANY": "syscall", - "syscall.SO_BINDTODEVICE": "syscall", - "syscall.SO_BINTIME": "syscall", - "syscall.SO_BROADCAST": "syscall", - "syscall.SO_BSDCOMPAT": "syscall", - "syscall.SO_DEBUG": "syscall", - "syscall.SO_DETACH_FILTER": "syscall", - "syscall.SO_DOMAIN": "syscall", - "syscall.SO_DONTROUTE": "syscall", - "syscall.SO_DONTTRUNC": "syscall", - "syscall.SO_ERROR": "syscall", - "syscall.SO_KEEPALIVE": "syscall", - "syscall.SO_LABEL": "syscall", - "syscall.SO_LINGER": "syscall", - "syscall.SO_LINGER_SEC": "syscall", - "syscall.SO_LISTENINCQLEN": "syscall", - "syscall.SO_LISTENQLEN": "syscall", - "syscall.SO_LISTENQLIMIT": "syscall", - "syscall.SO_MARK": "syscall", - "syscall.SO_NETPROC": "syscall", - "syscall.SO_NKE": "syscall", - "syscall.SO_NOADDRERR": "syscall", - "syscall.SO_NOHEADER": "syscall", - "syscall.SO_NOSIGPIPE": "syscall", - "syscall.SO_NOTIFYCONFLICT": "syscall", - "syscall.SO_NO_CHECK": "syscall", - "syscall.SO_NO_DDP": "syscall", - "syscall.SO_NO_OFFLOAD": "syscall", - "syscall.SO_NP_EXTENSIONS": "syscall", - "syscall.SO_NREAD": "syscall", - "syscall.SO_NWRITE": "syscall", - "syscall.SO_OOBINLINE": "syscall", - "syscall.SO_OVERFLOWED": "syscall", - "syscall.SO_PASSCRED": "syscall", - "syscall.SO_PASSSEC": "syscall", - "syscall.SO_PEERCRED": "syscall", - "syscall.SO_PEERLABEL": "syscall", - "syscall.SO_PEERNAME": "syscall", - "syscall.SO_PEERSEC": "syscall", - "syscall.SO_PRIORITY": "syscall", - "syscall.SO_PROTOCOL": "syscall", - "syscall.SO_PROTOTYPE": "syscall", - "syscall.SO_RANDOMPORT": "syscall", - "syscall.SO_RCVBUF": "syscall", - "syscall.SO_RCVBUFFORCE": "syscall", - "syscall.SO_RCVLOWAT": "syscall", - "syscall.SO_RCVTIMEO": "syscall", - "syscall.SO_RESTRICTIONS": "syscall", - "syscall.SO_RESTRICT_DENYIN": "syscall", - "syscall.SO_RESTRICT_DENYOUT": "syscall", - "syscall.SO_RESTRICT_DENYSET": "syscall", - "syscall.SO_REUSEADDR": "syscall", - "syscall.SO_REUSEPORT": "syscall", - "syscall.SO_REUSESHAREUID": "syscall", - "syscall.SO_RTABLE": "syscall", - "syscall.SO_RXQ_OVFL": "syscall", - "syscall.SO_SECURITY_AUTHENTICATION": "syscall", - "syscall.SO_SECURITY_ENCRYPTION_NETWORK": "syscall", - "syscall.SO_SECURITY_ENCRYPTION_TRANSPORT": "syscall", - "syscall.SO_SETFIB": "syscall", - "syscall.SO_SNDBUF": "syscall", - "syscall.SO_SNDBUFFORCE": "syscall", - "syscall.SO_SNDLOWAT": "syscall", - "syscall.SO_SNDTIMEO": "syscall", - "syscall.SO_SPLICE": "syscall", - "syscall.SO_TIMESTAMP": "syscall", - "syscall.SO_TIMESTAMPING": "syscall", - "syscall.SO_TIMESTAMPNS": "syscall", - "syscall.SO_TIMESTAMP_MONOTONIC": "syscall", - "syscall.SO_TYPE": "syscall", - "syscall.SO_UPCALLCLOSEWAIT": "syscall", - "syscall.SO_UPDATE_ACCEPT_CONTEXT": "syscall", - "syscall.SO_UPDATE_CONNECT_CONTEXT": "syscall", - "syscall.SO_USELOOPBACK": "syscall", - "syscall.SO_USER_COOKIE": "syscall", - "syscall.SO_VENDOR": "syscall", - "syscall.SO_WANTMORE": "syscall", - "syscall.SO_WANTOOBFLAG": "syscall", - "syscall.SSLExtraCertChainPolicyPara": "syscall", - "syscall.STANDARD_RIGHTS_ALL": "syscall", - "syscall.STANDARD_RIGHTS_EXECUTE": "syscall", - "syscall.STANDARD_RIGHTS_READ": "syscall", - "syscall.STANDARD_RIGHTS_REQUIRED": "syscall", - "syscall.STANDARD_RIGHTS_WRITE": "syscall", - "syscall.STARTF_USESHOWWINDOW": "syscall", - "syscall.STARTF_USESTDHANDLES": "syscall", - "syscall.STD_ERROR_HANDLE": "syscall", - "syscall.STD_INPUT_HANDLE": "syscall", - "syscall.STD_OUTPUT_HANDLE": "syscall", - "syscall.SUBLANG_ENGLISH_US": "syscall", - "syscall.SW_FORCEMINIMIZE": "syscall", - "syscall.SW_HIDE": "syscall", - "syscall.SW_MAXIMIZE": "syscall", - "syscall.SW_MINIMIZE": "syscall", - "syscall.SW_NORMAL": "syscall", - "syscall.SW_RESTORE": "syscall", - "syscall.SW_SHOW": "syscall", - "syscall.SW_SHOWDEFAULT": "syscall", - "syscall.SW_SHOWMAXIMIZED": "syscall", - "syscall.SW_SHOWMINIMIZED": "syscall", - "syscall.SW_SHOWMINNOACTIVE": "syscall", - "syscall.SW_SHOWNA": "syscall", - "syscall.SW_SHOWNOACTIVATE": "syscall", - "syscall.SW_SHOWNORMAL": "syscall", - "syscall.SYMBOLIC_LINK_FLAG_DIRECTORY": "syscall", - "syscall.SYNCHRONIZE": "syscall", - "syscall.SYSCTL_VERSION": "syscall", - "syscall.SYSCTL_VERS_0": "syscall", - "syscall.SYSCTL_VERS_1": "syscall", - "syscall.SYSCTL_VERS_MASK": "syscall", - "syscall.SYS_ABORT2": "syscall", - "syscall.SYS_ACCEPT": "syscall", - "syscall.SYS_ACCEPT4": "syscall", - "syscall.SYS_ACCEPT_NOCANCEL": "syscall", - "syscall.SYS_ACCESS": "syscall", - "syscall.SYS_ACCESS_EXTENDED": "syscall", - "syscall.SYS_ACCT": "syscall", - "syscall.SYS_ADD_KEY": "syscall", - "syscall.SYS_ADD_PROFIL": "syscall", - "syscall.SYS_ADJFREQ": "syscall", - "syscall.SYS_ADJTIME": "syscall", - "syscall.SYS_ADJTIMEX": "syscall", - "syscall.SYS_AFS_SYSCALL": "syscall", - "syscall.SYS_AIO_CANCEL": "syscall", - "syscall.SYS_AIO_ERROR": "syscall", - "syscall.SYS_AIO_FSYNC": "syscall", - "syscall.SYS_AIO_READ": "syscall", - "syscall.SYS_AIO_RETURN": "syscall", - "syscall.SYS_AIO_SUSPEND": "syscall", - "syscall.SYS_AIO_SUSPEND_NOCANCEL": "syscall", - "syscall.SYS_AIO_WRITE": "syscall", - "syscall.SYS_ALARM": "syscall", - "syscall.SYS_ARCH_PRCTL": "syscall", - "syscall.SYS_ARM_FADVISE64_64": "syscall", - "syscall.SYS_ARM_SYNC_FILE_RANGE": "syscall", - "syscall.SYS_ATGETMSG": "syscall", - "syscall.SYS_ATPGETREQ": "syscall", - "syscall.SYS_ATPGETRSP": "syscall", - "syscall.SYS_ATPSNDREQ": "syscall", - "syscall.SYS_ATPSNDRSP": "syscall", - "syscall.SYS_ATPUTMSG": "syscall", - "syscall.SYS_ATSOCKET": "syscall", - "syscall.SYS_AUDIT": "syscall", - "syscall.SYS_AUDITCTL": "syscall", - "syscall.SYS_AUDITON": "syscall", - "syscall.SYS_AUDIT_SESSION_JOIN": "syscall", - "syscall.SYS_AUDIT_SESSION_PORT": "syscall", - "syscall.SYS_AUDIT_SESSION_SELF": "syscall", - "syscall.SYS_BDFLUSH": "syscall", - "syscall.SYS_BIND": "syscall", - "syscall.SYS_BINDAT": "syscall", - "syscall.SYS_BREAK": "syscall", - "syscall.SYS_BRK": "syscall", - "syscall.SYS_BSDTHREAD_CREATE": "syscall", - "syscall.SYS_BSDTHREAD_REGISTER": "syscall", - "syscall.SYS_BSDTHREAD_TERMINATE": "syscall", - "syscall.SYS_CAPGET": "syscall", - "syscall.SYS_CAPSET": "syscall", - "syscall.SYS_CAP_ENTER": "syscall", - "syscall.SYS_CAP_FCNTLS_GET": "syscall", - "syscall.SYS_CAP_FCNTLS_LIMIT": "syscall", - "syscall.SYS_CAP_GETMODE": "syscall", - "syscall.SYS_CAP_GETRIGHTS": "syscall", - "syscall.SYS_CAP_IOCTLS_GET": "syscall", - "syscall.SYS_CAP_IOCTLS_LIMIT": "syscall", - "syscall.SYS_CAP_NEW": "syscall", - "syscall.SYS_CAP_RIGHTS_GET": "syscall", - "syscall.SYS_CAP_RIGHTS_LIMIT": "syscall", - "syscall.SYS_CHDIR": "syscall", - "syscall.SYS_CHFLAGS": "syscall", - "syscall.SYS_CHFLAGSAT": "syscall", - "syscall.SYS_CHMOD": "syscall", - "syscall.SYS_CHMOD_EXTENDED": "syscall", - "syscall.SYS_CHOWN": "syscall", - "syscall.SYS_CHOWN32": "syscall", - "syscall.SYS_CHROOT": "syscall", - "syscall.SYS_CHUD": "syscall", - "syscall.SYS_CLOCK_ADJTIME": "syscall", - "syscall.SYS_CLOCK_GETCPUCLOCKID2": "syscall", - "syscall.SYS_CLOCK_GETRES": "syscall", - "syscall.SYS_CLOCK_GETTIME": "syscall", - "syscall.SYS_CLOCK_NANOSLEEP": "syscall", - "syscall.SYS_CLOCK_SETTIME": "syscall", - "syscall.SYS_CLONE": "syscall", - "syscall.SYS_CLOSE": "syscall", - "syscall.SYS_CLOSEFROM": "syscall", - "syscall.SYS_CLOSE_NOCANCEL": "syscall", - "syscall.SYS_CONNECT": "syscall", - "syscall.SYS_CONNECTAT": "syscall", - "syscall.SYS_CONNECT_NOCANCEL": "syscall", - "syscall.SYS_COPYFILE": "syscall", - "syscall.SYS_CPUSET": "syscall", - "syscall.SYS_CPUSET_GETAFFINITY": "syscall", - "syscall.SYS_CPUSET_GETID": "syscall", - "syscall.SYS_CPUSET_SETAFFINITY": "syscall", - "syscall.SYS_CPUSET_SETID": "syscall", - "syscall.SYS_CREAT": "syscall", - "syscall.SYS_CREATE_MODULE": "syscall", - "syscall.SYS_CSOPS": "syscall", - "syscall.SYS_DELETE": "syscall", - "syscall.SYS_DELETE_MODULE": "syscall", - "syscall.SYS_DUP": "syscall", - "syscall.SYS_DUP2": "syscall", - "syscall.SYS_DUP3": "syscall", - "syscall.SYS_EACCESS": "syscall", - "syscall.SYS_EPOLL_CREATE": "syscall", - "syscall.SYS_EPOLL_CREATE1": "syscall", - "syscall.SYS_EPOLL_CTL": "syscall", - "syscall.SYS_EPOLL_CTL_OLD": "syscall", - "syscall.SYS_EPOLL_PWAIT": "syscall", - "syscall.SYS_EPOLL_WAIT": "syscall", - "syscall.SYS_EPOLL_WAIT_OLD": "syscall", - "syscall.SYS_EVENTFD": "syscall", - "syscall.SYS_EVENTFD2": "syscall", - "syscall.SYS_EXCHANGEDATA": "syscall", - "syscall.SYS_EXECVE": "syscall", - "syscall.SYS_EXIT": "syscall", - "syscall.SYS_EXIT_GROUP": "syscall", - "syscall.SYS_EXTATTRCTL": "syscall", - "syscall.SYS_EXTATTR_DELETE_FD": "syscall", - "syscall.SYS_EXTATTR_DELETE_FILE": "syscall", - "syscall.SYS_EXTATTR_DELETE_LINK": "syscall", - "syscall.SYS_EXTATTR_GET_FD": "syscall", - "syscall.SYS_EXTATTR_GET_FILE": "syscall", - "syscall.SYS_EXTATTR_GET_LINK": "syscall", - "syscall.SYS_EXTATTR_LIST_FD": "syscall", - "syscall.SYS_EXTATTR_LIST_FILE": "syscall", - "syscall.SYS_EXTATTR_LIST_LINK": "syscall", - "syscall.SYS_EXTATTR_SET_FD": "syscall", - "syscall.SYS_EXTATTR_SET_FILE": "syscall", - "syscall.SYS_EXTATTR_SET_LINK": "syscall", - "syscall.SYS_FACCESSAT": "syscall", - "syscall.SYS_FADVISE64": "syscall", - "syscall.SYS_FADVISE64_64": "syscall", - "syscall.SYS_FALLOCATE": "syscall", - "syscall.SYS_FANOTIFY_INIT": "syscall", - "syscall.SYS_FANOTIFY_MARK": "syscall", - "syscall.SYS_FCHDIR": "syscall", - "syscall.SYS_FCHFLAGS": "syscall", - "syscall.SYS_FCHMOD": "syscall", - "syscall.SYS_FCHMODAT": "syscall", - "syscall.SYS_FCHMOD_EXTENDED": "syscall", - "syscall.SYS_FCHOWN": "syscall", - "syscall.SYS_FCHOWN32": "syscall", - "syscall.SYS_FCHOWNAT": "syscall", - "syscall.SYS_FCHROOT": "syscall", - "syscall.SYS_FCNTL": "syscall", - "syscall.SYS_FCNTL64": "syscall", - "syscall.SYS_FCNTL_NOCANCEL": "syscall", - "syscall.SYS_FDATASYNC": "syscall", - "syscall.SYS_FEXECVE": "syscall", - "syscall.SYS_FFCLOCK_GETCOUNTER": "syscall", - "syscall.SYS_FFCLOCK_GETESTIMATE": "syscall", - "syscall.SYS_FFCLOCK_SETESTIMATE": "syscall", - "syscall.SYS_FFSCTL": "syscall", - "syscall.SYS_FGETATTRLIST": "syscall", - "syscall.SYS_FGETXATTR": "syscall", - "syscall.SYS_FHOPEN": "syscall", - "syscall.SYS_FHSTAT": "syscall", - "syscall.SYS_FHSTATFS": "syscall", - "syscall.SYS_FILEPORT_MAKEFD": "syscall", - "syscall.SYS_FILEPORT_MAKEPORT": "syscall", - "syscall.SYS_FKTRACE": "syscall", - "syscall.SYS_FLISTXATTR": "syscall", - "syscall.SYS_FLOCK": "syscall", - "syscall.SYS_FORK": "syscall", - "syscall.SYS_FPATHCONF": "syscall", - "syscall.SYS_FREEBSD6_FTRUNCATE": "syscall", - "syscall.SYS_FREEBSD6_LSEEK": "syscall", - "syscall.SYS_FREEBSD6_MMAP": "syscall", - "syscall.SYS_FREEBSD6_PREAD": "syscall", - "syscall.SYS_FREEBSD6_PWRITE": "syscall", - "syscall.SYS_FREEBSD6_TRUNCATE": "syscall", - "syscall.SYS_FREMOVEXATTR": "syscall", - "syscall.SYS_FSCTL": "syscall", - "syscall.SYS_FSETATTRLIST": "syscall", - "syscall.SYS_FSETXATTR": "syscall", - "syscall.SYS_FSGETPATH": "syscall", - "syscall.SYS_FSTAT": "syscall", - "syscall.SYS_FSTAT64": "syscall", - "syscall.SYS_FSTAT64_EXTENDED": "syscall", - "syscall.SYS_FSTATAT": "syscall", - "syscall.SYS_FSTATAT64": "syscall", - "syscall.SYS_FSTATFS": "syscall", - "syscall.SYS_FSTATFS64": "syscall", - "syscall.SYS_FSTATV": "syscall", - "syscall.SYS_FSTATVFS1": "syscall", - "syscall.SYS_FSTAT_EXTENDED": "syscall", - "syscall.SYS_FSYNC": "syscall", - "syscall.SYS_FSYNC_NOCANCEL": "syscall", - "syscall.SYS_FSYNC_RANGE": "syscall", - "syscall.SYS_FTIME": "syscall", - "syscall.SYS_FTRUNCATE": "syscall", - "syscall.SYS_FTRUNCATE64": "syscall", - "syscall.SYS_FUTEX": "syscall", - "syscall.SYS_FUTIMENS": "syscall", - "syscall.SYS_FUTIMES": "syscall", - "syscall.SYS_FUTIMESAT": "syscall", - "syscall.SYS_GETATTRLIST": "syscall", - "syscall.SYS_GETAUDIT": "syscall", - "syscall.SYS_GETAUDIT_ADDR": "syscall", - "syscall.SYS_GETAUID": "syscall", - "syscall.SYS_GETCONTEXT": "syscall", - "syscall.SYS_GETCPU": "syscall", - "syscall.SYS_GETCWD": "syscall", - "syscall.SYS_GETDENTS": "syscall", - "syscall.SYS_GETDENTS64": "syscall", - "syscall.SYS_GETDIRENTRIES": "syscall", - "syscall.SYS_GETDIRENTRIES64": "syscall", - "syscall.SYS_GETDIRENTRIESATTR": "syscall", - "syscall.SYS_GETDTABLECOUNT": "syscall", - "syscall.SYS_GETDTABLESIZE": "syscall", - "syscall.SYS_GETEGID": "syscall", - "syscall.SYS_GETEGID32": "syscall", - "syscall.SYS_GETEUID": "syscall", - "syscall.SYS_GETEUID32": "syscall", - "syscall.SYS_GETFH": "syscall", - "syscall.SYS_GETFSSTAT": "syscall", - "syscall.SYS_GETFSSTAT64": "syscall", - "syscall.SYS_GETGID": "syscall", - "syscall.SYS_GETGID32": "syscall", - "syscall.SYS_GETGROUPS": "syscall", - "syscall.SYS_GETGROUPS32": "syscall", - "syscall.SYS_GETHOSTUUID": "syscall", - "syscall.SYS_GETITIMER": "syscall", - "syscall.SYS_GETLCID": "syscall", - "syscall.SYS_GETLOGIN": "syscall", - "syscall.SYS_GETLOGINCLASS": "syscall", - "syscall.SYS_GETPEERNAME": "syscall", - "syscall.SYS_GETPGID": "syscall", - "syscall.SYS_GETPGRP": "syscall", - "syscall.SYS_GETPID": "syscall", - "syscall.SYS_GETPMSG": "syscall", - "syscall.SYS_GETPPID": "syscall", - "syscall.SYS_GETPRIORITY": "syscall", - "syscall.SYS_GETRESGID": "syscall", - "syscall.SYS_GETRESGID32": "syscall", - "syscall.SYS_GETRESUID": "syscall", - "syscall.SYS_GETRESUID32": "syscall", - "syscall.SYS_GETRLIMIT": "syscall", - "syscall.SYS_GETRTABLE": "syscall", - "syscall.SYS_GETRUSAGE": "syscall", - "syscall.SYS_GETSGROUPS": "syscall", - "syscall.SYS_GETSID": "syscall", - "syscall.SYS_GETSOCKNAME": "syscall", - "syscall.SYS_GETSOCKOPT": "syscall", - "syscall.SYS_GETTHRID": "syscall", - "syscall.SYS_GETTID": "syscall", - "syscall.SYS_GETTIMEOFDAY": "syscall", - "syscall.SYS_GETUID": "syscall", - "syscall.SYS_GETUID32": "syscall", - "syscall.SYS_GETVFSSTAT": "syscall", - "syscall.SYS_GETWGROUPS": "syscall", - "syscall.SYS_GETXATTR": "syscall", - "syscall.SYS_GET_KERNEL_SYMS": "syscall", - "syscall.SYS_GET_MEMPOLICY": "syscall", - "syscall.SYS_GET_ROBUST_LIST": "syscall", - "syscall.SYS_GET_THREAD_AREA": "syscall", - "syscall.SYS_GTTY": "syscall", - "syscall.SYS_IDENTITYSVC": "syscall", - "syscall.SYS_IDLE": "syscall", - "syscall.SYS_INITGROUPS": "syscall", - "syscall.SYS_INIT_MODULE": "syscall", - "syscall.SYS_INOTIFY_ADD_WATCH": "syscall", - "syscall.SYS_INOTIFY_INIT": "syscall", - "syscall.SYS_INOTIFY_INIT1": "syscall", - "syscall.SYS_INOTIFY_RM_WATCH": "syscall", - "syscall.SYS_IOCTL": "syscall", - "syscall.SYS_IOPERM": "syscall", - "syscall.SYS_IOPL": "syscall", - "syscall.SYS_IOPOLICYSYS": "syscall", - "syscall.SYS_IOPRIO_GET": "syscall", - "syscall.SYS_IOPRIO_SET": "syscall", - "syscall.SYS_IO_CANCEL": "syscall", - "syscall.SYS_IO_DESTROY": "syscall", - "syscall.SYS_IO_GETEVENTS": "syscall", - "syscall.SYS_IO_SETUP": "syscall", - "syscall.SYS_IO_SUBMIT": "syscall", - "syscall.SYS_IPC": "syscall", - "syscall.SYS_ISSETUGID": "syscall", - "syscall.SYS_JAIL": "syscall", - "syscall.SYS_JAIL_ATTACH": "syscall", - "syscall.SYS_JAIL_GET": "syscall", - "syscall.SYS_JAIL_REMOVE": "syscall", - "syscall.SYS_JAIL_SET": "syscall", - "syscall.SYS_KDEBUG_TRACE": "syscall", - "syscall.SYS_KENV": "syscall", - "syscall.SYS_KEVENT": "syscall", - "syscall.SYS_KEVENT64": "syscall", - "syscall.SYS_KEXEC_LOAD": "syscall", - "syscall.SYS_KEYCTL": "syscall", - "syscall.SYS_KILL": "syscall", - "syscall.SYS_KLDFIND": "syscall", - "syscall.SYS_KLDFIRSTMOD": "syscall", - "syscall.SYS_KLDLOAD": "syscall", - "syscall.SYS_KLDNEXT": "syscall", - "syscall.SYS_KLDSTAT": "syscall", - "syscall.SYS_KLDSYM": "syscall", - "syscall.SYS_KLDUNLOAD": "syscall", - "syscall.SYS_KLDUNLOADF": "syscall", - "syscall.SYS_KQUEUE": "syscall", - "syscall.SYS_KQUEUE1": "syscall", - "syscall.SYS_KTIMER_CREATE": "syscall", - "syscall.SYS_KTIMER_DELETE": "syscall", - "syscall.SYS_KTIMER_GETOVERRUN": "syscall", - "syscall.SYS_KTIMER_GETTIME": "syscall", - "syscall.SYS_KTIMER_SETTIME": "syscall", - "syscall.SYS_KTRACE": "syscall", - "syscall.SYS_LCHFLAGS": "syscall", - "syscall.SYS_LCHMOD": "syscall", - "syscall.SYS_LCHOWN": "syscall", - "syscall.SYS_LCHOWN32": "syscall", - "syscall.SYS_LGETFH": "syscall", - "syscall.SYS_LGETXATTR": "syscall", - "syscall.SYS_LINK": "syscall", - "syscall.SYS_LINKAT": "syscall", - "syscall.SYS_LIO_LISTIO": "syscall", - "syscall.SYS_LISTEN": "syscall", - "syscall.SYS_LISTXATTR": "syscall", - "syscall.SYS_LLISTXATTR": "syscall", - "syscall.SYS_LOCK": "syscall", - "syscall.SYS_LOOKUP_DCOOKIE": "syscall", - "syscall.SYS_LPATHCONF": "syscall", - "syscall.SYS_LREMOVEXATTR": "syscall", - "syscall.SYS_LSEEK": "syscall", - "syscall.SYS_LSETXATTR": "syscall", - "syscall.SYS_LSTAT": "syscall", - "syscall.SYS_LSTAT64": "syscall", - "syscall.SYS_LSTAT64_EXTENDED": "syscall", - "syscall.SYS_LSTATV": "syscall", - "syscall.SYS_LSTAT_EXTENDED": "syscall", - "syscall.SYS_LUTIMES": "syscall", - "syscall.SYS_MAC_SYSCALL": "syscall", - "syscall.SYS_MADVISE": "syscall", - "syscall.SYS_MADVISE1": "syscall", - "syscall.SYS_MAXSYSCALL": "syscall", - "syscall.SYS_MBIND": "syscall", - "syscall.SYS_MIGRATE_PAGES": "syscall", - "syscall.SYS_MINCORE": "syscall", - "syscall.SYS_MINHERIT": "syscall", - "syscall.SYS_MKCOMPLEX": "syscall", - "syscall.SYS_MKDIR": "syscall", - "syscall.SYS_MKDIRAT": "syscall", - "syscall.SYS_MKDIR_EXTENDED": "syscall", - "syscall.SYS_MKFIFO": "syscall", - "syscall.SYS_MKFIFOAT": "syscall", - "syscall.SYS_MKFIFO_EXTENDED": "syscall", - "syscall.SYS_MKNOD": "syscall", - "syscall.SYS_MKNODAT": "syscall", - "syscall.SYS_MLOCK": "syscall", - "syscall.SYS_MLOCKALL": "syscall", - "syscall.SYS_MMAP": "syscall", - "syscall.SYS_MMAP2": "syscall", - "syscall.SYS_MODCTL": "syscall", - "syscall.SYS_MODFIND": "syscall", - "syscall.SYS_MODFNEXT": "syscall", - "syscall.SYS_MODIFY_LDT": "syscall", - "syscall.SYS_MODNEXT": "syscall", - "syscall.SYS_MODSTAT": "syscall", - "syscall.SYS_MODWATCH": "syscall", - "syscall.SYS_MOUNT": "syscall", - "syscall.SYS_MOVE_PAGES": "syscall", - "syscall.SYS_MPROTECT": "syscall", - "syscall.SYS_MPX": "syscall", - "syscall.SYS_MQUERY": "syscall", - "syscall.SYS_MQ_GETSETATTR": "syscall", - "syscall.SYS_MQ_NOTIFY": "syscall", - "syscall.SYS_MQ_OPEN": "syscall", - "syscall.SYS_MQ_TIMEDRECEIVE": "syscall", - "syscall.SYS_MQ_TIMEDSEND": "syscall", - "syscall.SYS_MQ_UNLINK": "syscall", - "syscall.SYS_MREMAP": "syscall", - "syscall.SYS_MSGCTL": "syscall", - "syscall.SYS_MSGGET": "syscall", - "syscall.SYS_MSGRCV": "syscall", - "syscall.SYS_MSGRCV_NOCANCEL": "syscall", - "syscall.SYS_MSGSND": "syscall", - "syscall.SYS_MSGSND_NOCANCEL": "syscall", - "syscall.SYS_MSGSYS": "syscall", - "syscall.SYS_MSYNC": "syscall", - "syscall.SYS_MSYNC_NOCANCEL": "syscall", - "syscall.SYS_MUNLOCK": "syscall", - "syscall.SYS_MUNLOCKALL": "syscall", - "syscall.SYS_MUNMAP": "syscall", - "syscall.SYS_NAME_TO_HANDLE_AT": "syscall", - "syscall.SYS_NANOSLEEP": "syscall", - "syscall.SYS_NEWFSTATAT": "syscall", - "syscall.SYS_NFSCLNT": "syscall", - "syscall.SYS_NFSSERVCTL": "syscall", - "syscall.SYS_NFSSVC": "syscall", - "syscall.SYS_NFSTAT": "syscall", - "syscall.SYS_NICE": "syscall", - "syscall.SYS_NLSTAT": "syscall", - "syscall.SYS_NMOUNT": "syscall", - "syscall.SYS_NSTAT": "syscall", - "syscall.SYS_NTP_ADJTIME": "syscall", - "syscall.SYS_NTP_GETTIME": "syscall", - "syscall.SYS_OABI_SYSCALL_BASE": "syscall", - "syscall.SYS_OBREAK": "syscall", - "syscall.SYS_OLDFSTAT": "syscall", - "syscall.SYS_OLDLSTAT": "syscall", - "syscall.SYS_OLDOLDUNAME": "syscall", - "syscall.SYS_OLDSTAT": "syscall", - "syscall.SYS_OLDUNAME": "syscall", - "syscall.SYS_OPEN": "syscall", - "syscall.SYS_OPENAT": "syscall", - "syscall.SYS_OPENBSD_POLL": "syscall", - "syscall.SYS_OPEN_BY_HANDLE_AT": "syscall", - "syscall.SYS_OPEN_EXTENDED": "syscall", - "syscall.SYS_OPEN_NOCANCEL": "syscall", - "syscall.SYS_OVADVISE": "syscall", - "syscall.SYS_PACCEPT": "syscall", - "syscall.SYS_PATHCONF": "syscall", - "syscall.SYS_PAUSE": "syscall", - "syscall.SYS_PCICONFIG_IOBASE": "syscall", - "syscall.SYS_PCICONFIG_READ": "syscall", - "syscall.SYS_PCICONFIG_WRITE": "syscall", - "syscall.SYS_PDFORK": "syscall", - "syscall.SYS_PDGETPID": "syscall", - "syscall.SYS_PDKILL": "syscall", - "syscall.SYS_PERF_EVENT_OPEN": "syscall", - "syscall.SYS_PERSONALITY": "syscall", - "syscall.SYS_PID_HIBERNATE": "syscall", - "syscall.SYS_PID_RESUME": "syscall", - "syscall.SYS_PID_SHUTDOWN_SOCKETS": "syscall", - "syscall.SYS_PID_SUSPEND": "syscall", - "syscall.SYS_PIPE": "syscall", - "syscall.SYS_PIPE2": "syscall", - "syscall.SYS_PIVOT_ROOT": "syscall", - "syscall.SYS_PMC_CONTROL": "syscall", - "syscall.SYS_PMC_GET_INFO": "syscall", - "syscall.SYS_POLL": "syscall", - "syscall.SYS_POLLTS": "syscall", - "syscall.SYS_POLL_NOCANCEL": "syscall", - "syscall.SYS_POSIX_FADVISE": "syscall", - "syscall.SYS_POSIX_FALLOCATE": "syscall", - "syscall.SYS_POSIX_OPENPT": "syscall", - "syscall.SYS_POSIX_SPAWN": "syscall", - "syscall.SYS_PPOLL": "syscall", - "syscall.SYS_PRCTL": "syscall", - "syscall.SYS_PREAD": "syscall", - "syscall.SYS_PREAD64": "syscall", - "syscall.SYS_PREADV": "syscall", - "syscall.SYS_PREAD_NOCANCEL": "syscall", - "syscall.SYS_PRLIMIT64": "syscall", - "syscall.SYS_PROCCTL": "syscall", - "syscall.SYS_PROCESS_POLICY": "syscall", - "syscall.SYS_PROCESS_VM_READV": "syscall", - "syscall.SYS_PROCESS_VM_WRITEV": "syscall", - "syscall.SYS_PROC_INFO": "syscall", - "syscall.SYS_PROF": "syscall", - "syscall.SYS_PROFIL": "syscall", - "syscall.SYS_PSELECT": "syscall", - "syscall.SYS_PSELECT6": "syscall", - "syscall.SYS_PSET_ASSIGN": "syscall", - "syscall.SYS_PSET_CREATE": "syscall", - "syscall.SYS_PSET_DESTROY": "syscall", - "syscall.SYS_PSYNCH_CVBROAD": "syscall", - "syscall.SYS_PSYNCH_CVCLRPREPOST": "syscall", - "syscall.SYS_PSYNCH_CVSIGNAL": "syscall", - "syscall.SYS_PSYNCH_CVWAIT": "syscall", - "syscall.SYS_PSYNCH_MUTEXDROP": "syscall", - "syscall.SYS_PSYNCH_MUTEXWAIT": "syscall", - "syscall.SYS_PSYNCH_RW_DOWNGRADE": "syscall", - "syscall.SYS_PSYNCH_RW_LONGRDLOCK": "syscall", - "syscall.SYS_PSYNCH_RW_RDLOCK": "syscall", - "syscall.SYS_PSYNCH_RW_UNLOCK": "syscall", - "syscall.SYS_PSYNCH_RW_UNLOCK2": "syscall", - "syscall.SYS_PSYNCH_RW_UPGRADE": "syscall", - "syscall.SYS_PSYNCH_RW_WRLOCK": "syscall", - "syscall.SYS_PSYNCH_RW_YIELDWRLOCK": "syscall", - "syscall.SYS_PTRACE": "syscall", - "syscall.SYS_PUTPMSG": "syscall", - "syscall.SYS_PWRITE": "syscall", - "syscall.SYS_PWRITE64": "syscall", - "syscall.SYS_PWRITEV": "syscall", - "syscall.SYS_PWRITE_NOCANCEL": "syscall", - "syscall.SYS_QUERY_MODULE": "syscall", - "syscall.SYS_QUOTACTL": "syscall", - "syscall.SYS_RASCTL": "syscall", - "syscall.SYS_RCTL_ADD_RULE": "syscall", - "syscall.SYS_RCTL_GET_LIMITS": "syscall", - "syscall.SYS_RCTL_GET_RACCT": "syscall", - "syscall.SYS_RCTL_GET_RULES": "syscall", - "syscall.SYS_RCTL_REMOVE_RULE": "syscall", - "syscall.SYS_READ": "syscall", - "syscall.SYS_READAHEAD": "syscall", - "syscall.SYS_READDIR": "syscall", - "syscall.SYS_READLINK": "syscall", - "syscall.SYS_READLINKAT": "syscall", - "syscall.SYS_READV": "syscall", - "syscall.SYS_READV_NOCANCEL": "syscall", - "syscall.SYS_READ_NOCANCEL": "syscall", - "syscall.SYS_REBOOT": "syscall", - "syscall.SYS_RECV": "syscall", - "syscall.SYS_RECVFROM": "syscall", - "syscall.SYS_RECVFROM_NOCANCEL": "syscall", - "syscall.SYS_RECVMMSG": "syscall", - "syscall.SYS_RECVMSG": "syscall", - "syscall.SYS_RECVMSG_NOCANCEL": "syscall", - "syscall.SYS_REMAP_FILE_PAGES": "syscall", - "syscall.SYS_REMOVEXATTR": "syscall", - "syscall.SYS_RENAME": "syscall", - "syscall.SYS_RENAMEAT": "syscall", - "syscall.SYS_REQUEST_KEY": "syscall", - "syscall.SYS_RESTART_SYSCALL": "syscall", - "syscall.SYS_REVOKE": "syscall", - "syscall.SYS_RFORK": "syscall", - "syscall.SYS_RMDIR": "syscall", - "syscall.SYS_RTPRIO": "syscall", - "syscall.SYS_RTPRIO_THREAD": "syscall", - "syscall.SYS_RT_SIGACTION": "syscall", - "syscall.SYS_RT_SIGPENDING": "syscall", - "syscall.SYS_RT_SIGPROCMASK": "syscall", - "syscall.SYS_RT_SIGQUEUEINFO": "syscall", - "syscall.SYS_RT_SIGRETURN": "syscall", - "syscall.SYS_RT_SIGSUSPEND": "syscall", - "syscall.SYS_RT_SIGTIMEDWAIT": "syscall", - "syscall.SYS_RT_TGSIGQUEUEINFO": "syscall", - "syscall.SYS_SBRK": "syscall", - "syscall.SYS_SCHED_GETAFFINITY": "syscall", - "syscall.SYS_SCHED_GETPARAM": "syscall", - "syscall.SYS_SCHED_GETSCHEDULER": "syscall", - "syscall.SYS_SCHED_GET_PRIORITY_MAX": "syscall", - "syscall.SYS_SCHED_GET_PRIORITY_MIN": "syscall", - "syscall.SYS_SCHED_RR_GET_INTERVAL": "syscall", - "syscall.SYS_SCHED_SETAFFINITY": "syscall", - "syscall.SYS_SCHED_SETPARAM": "syscall", - "syscall.SYS_SCHED_SETSCHEDULER": "syscall", - "syscall.SYS_SCHED_YIELD": "syscall", - "syscall.SYS_SCTP_GENERIC_RECVMSG": "syscall", - "syscall.SYS_SCTP_GENERIC_SENDMSG": "syscall", - "syscall.SYS_SCTP_GENERIC_SENDMSG_IOV": "syscall", - "syscall.SYS_SCTP_PEELOFF": "syscall", - "syscall.SYS_SEARCHFS": "syscall", - "syscall.SYS_SECURITY": "syscall", - "syscall.SYS_SELECT": "syscall", - "syscall.SYS_SELECT_NOCANCEL": "syscall", - "syscall.SYS_SEMCONFIG": "syscall", - "syscall.SYS_SEMCTL": "syscall", - "syscall.SYS_SEMGET": "syscall", - "syscall.SYS_SEMOP": "syscall", - "syscall.SYS_SEMSYS": "syscall", - "syscall.SYS_SEMTIMEDOP": "syscall", - "syscall.SYS_SEM_CLOSE": "syscall", - "syscall.SYS_SEM_DESTROY": "syscall", - "syscall.SYS_SEM_GETVALUE": "syscall", - "syscall.SYS_SEM_INIT": "syscall", - "syscall.SYS_SEM_OPEN": "syscall", - "syscall.SYS_SEM_POST": "syscall", - "syscall.SYS_SEM_TRYWAIT": "syscall", - "syscall.SYS_SEM_UNLINK": "syscall", - "syscall.SYS_SEM_WAIT": "syscall", - "syscall.SYS_SEM_WAIT_NOCANCEL": "syscall", - "syscall.SYS_SEND": "syscall", - "syscall.SYS_SENDFILE": "syscall", - "syscall.SYS_SENDFILE64": "syscall", - "syscall.SYS_SENDMMSG": "syscall", - "syscall.SYS_SENDMSG": "syscall", - "syscall.SYS_SENDMSG_NOCANCEL": "syscall", - "syscall.SYS_SENDTO": "syscall", - "syscall.SYS_SENDTO_NOCANCEL": "syscall", - "syscall.SYS_SETATTRLIST": "syscall", - "syscall.SYS_SETAUDIT": "syscall", - "syscall.SYS_SETAUDIT_ADDR": "syscall", - "syscall.SYS_SETAUID": "syscall", - "syscall.SYS_SETCONTEXT": "syscall", - "syscall.SYS_SETDOMAINNAME": "syscall", - "syscall.SYS_SETEGID": "syscall", - "syscall.SYS_SETEUID": "syscall", - "syscall.SYS_SETFIB": "syscall", - "syscall.SYS_SETFSGID": "syscall", - "syscall.SYS_SETFSGID32": "syscall", - "syscall.SYS_SETFSUID": "syscall", - "syscall.SYS_SETFSUID32": "syscall", - "syscall.SYS_SETGID": "syscall", - "syscall.SYS_SETGID32": "syscall", - "syscall.SYS_SETGROUPS": "syscall", - "syscall.SYS_SETGROUPS32": "syscall", - "syscall.SYS_SETHOSTNAME": "syscall", - "syscall.SYS_SETITIMER": "syscall", - "syscall.SYS_SETLCID": "syscall", - "syscall.SYS_SETLOGIN": "syscall", - "syscall.SYS_SETLOGINCLASS": "syscall", - "syscall.SYS_SETNS": "syscall", - "syscall.SYS_SETPGID": "syscall", - "syscall.SYS_SETPRIORITY": "syscall", - "syscall.SYS_SETPRIVEXEC": "syscall", - "syscall.SYS_SETREGID": "syscall", - "syscall.SYS_SETREGID32": "syscall", - "syscall.SYS_SETRESGID": "syscall", - "syscall.SYS_SETRESGID32": "syscall", - "syscall.SYS_SETRESUID": "syscall", - "syscall.SYS_SETRESUID32": "syscall", - "syscall.SYS_SETREUID": "syscall", - "syscall.SYS_SETREUID32": "syscall", - "syscall.SYS_SETRLIMIT": "syscall", - "syscall.SYS_SETRTABLE": "syscall", - "syscall.SYS_SETSGROUPS": "syscall", - "syscall.SYS_SETSID": "syscall", - "syscall.SYS_SETSOCKOPT": "syscall", - "syscall.SYS_SETTID": "syscall", - "syscall.SYS_SETTID_WITH_PID": "syscall", - "syscall.SYS_SETTIMEOFDAY": "syscall", - "syscall.SYS_SETUID": "syscall", - "syscall.SYS_SETUID32": "syscall", - "syscall.SYS_SETWGROUPS": "syscall", - "syscall.SYS_SETXATTR": "syscall", - "syscall.SYS_SET_MEMPOLICY": "syscall", - "syscall.SYS_SET_ROBUST_LIST": "syscall", - "syscall.SYS_SET_THREAD_AREA": "syscall", - "syscall.SYS_SET_TID_ADDRESS": "syscall", - "syscall.SYS_SGETMASK": "syscall", - "syscall.SYS_SHARED_REGION_CHECK_NP": "syscall", - "syscall.SYS_SHARED_REGION_MAP_AND_SLIDE_NP": "syscall", - "syscall.SYS_SHMAT": "syscall", - "syscall.SYS_SHMCTL": "syscall", - "syscall.SYS_SHMDT": "syscall", - "syscall.SYS_SHMGET": "syscall", - "syscall.SYS_SHMSYS": "syscall", - "syscall.SYS_SHM_OPEN": "syscall", - "syscall.SYS_SHM_UNLINK": "syscall", - "syscall.SYS_SHUTDOWN": "syscall", - "syscall.SYS_SIGACTION": "syscall", - "syscall.SYS_SIGALTSTACK": "syscall", - "syscall.SYS_SIGNAL": "syscall", - "syscall.SYS_SIGNALFD": "syscall", - "syscall.SYS_SIGNALFD4": "syscall", - "syscall.SYS_SIGPENDING": "syscall", - "syscall.SYS_SIGPROCMASK": "syscall", - "syscall.SYS_SIGQUEUE": "syscall", - "syscall.SYS_SIGQUEUEINFO": "syscall", - "syscall.SYS_SIGRETURN": "syscall", - "syscall.SYS_SIGSUSPEND": "syscall", - "syscall.SYS_SIGSUSPEND_NOCANCEL": "syscall", - "syscall.SYS_SIGTIMEDWAIT": "syscall", - "syscall.SYS_SIGWAIT": "syscall", - "syscall.SYS_SIGWAITINFO": "syscall", - "syscall.SYS_SOCKET": "syscall", - "syscall.SYS_SOCKETCALL": "syscall", - "syscall.SYS_SOCKETPAIR": "syscall", - "syscall.SYS_SPLICE": "syscall", - "syscall.SYS_SSETMASK": "syscall", - "syscall.SYS_SSTK": "syscall", - "syscall.SYS_STACK_SNAPSHOT": "syscall", - "syscall.SYS_STAT": "syscall", - "syscall.SYS_STAT64": "syscall", - "syscall.SYS_STAT64_EXTENDED": "syscall", - "syscall.SYS_STATFS": "syscall", - "syscall.SYS_STATFS64": "syscall", - "syscall.SYS_STATV": "syscall", - "syscall.SYS_STATVFS1": "syscall", - "syscall.SYS_STAT_EXTENDED": "syscall", - "syscall.SYS_STIME": "syscall", - "syscall.SYS_STTY": "syscall", - "syscall.SYS_SWAPCONTEXT": "syscall", - "syscall.SYS_SWAPCTL": "syscall", - "syscall.SYS_SWAPOFF": "syscall", - "syscall.SYS_SWAPON": "syscall", - "syscall.SYS_SYMLINK": "syscall", - "syscall.SYS_SYMLINKAT": "syscall", - "syscall.SYS_SYNC": "syscall", - "syscall.SYS_SYNCFS": "syscall", - "syscall.SYS_SYNC_FILE_RANGE": "syscall", - "syscall.SYS_SYSARCH": "syscall", - "syscall.SYS_SYSCALL": "syscall", - "syscall.SYS_SYSCALL_BASE": "syscall", - "syscall.SYS_SYSFS": "syscall", - "syscall.SYS_SYSINFO": "syscall", - "syscall.SYS_SYSLOG": "syscall", - "syscall.SYS_TEE": "syscall", - "syscall.SYS_TGKILL": "syscall", - "syscall.SYS_THREAD_SELFID": "syscall", - "syscall.SYS_THR_CREATE": "syscall", - "syscall.SYS_THR_EXIT": "syscall", - "syscall.SYS_THR_KILL": "syscall", - "syscall.SYS_THR_KILL2": "syscall", - "syscall.SYS_THR_NEW": "syscall", - "syscall.SYS_THR_SELF": "syscall", - "syscall.SYS_THR_SET_NAME": "syscall", - "syscall.SYS_THR_SUSPEND": "syscall", - "syscall.SYS_THR_WAKE": "syscall", - "syscall.SYS_TIME": "syscall", - "syscall.SYS_TIMERFD_CREATE": "syscall", - "syscall.SYS_TIMERFD_GETTIME": "syscall", - "syscall.SYS_TIMERFD_SETTIME": "syscall", - "syscall.SYS_TIMER_CREATE": "syscall", - "syscall.SYS_TIMER_DELETE": "syscall", - "syscall.SYS_TIMER_GETOVERRUN": "syscall", - "syscall.SYS_TIMER_GETTIME": "syscall", - "syscall.SYS_TIMER_SETTIME": "syscall", - "syscall.SYS_TIMES": "syscall", - "syscall.SYS_TKILL": "syscall", - "syscall.SYS_TRUNCATE": "syscall", - "syscall.SYS_TRUNCATE64": "syscall", - "syscall.SYS_TUXCALL": "syscall", - "syscall.SYS_UGETRLIMIT": "syscall", - "syscall.SYS_ULIMIT": "syscall", - "syscall.SYS_UMASK": "syscall", - "syscall.SYS_UMASK_EXTENDED": "syscall", - "syscall.SYS_UMOUNT": "syscall", - "syscall.SYS_UMOUNT2": "syscall", - "syscall.SYS_UNAME": "syscall", - "syscall.SYS_UNDELETE": "syscall", - "syscall.SYS_UNLINK": "syscall", - "syscall.SYS_UNLINKAT": "syscall", - "syscall.SYS_UNMOUNT": "syscall", - "syscall.SYS_UNSHARE": "syscall", - "syscall.SYS_USELIB": "syscall", - "syscall.SYS_USTAT": "syscall", - "syscall.SYS_UTIME": "syscall", - "syscall.SYS_UTIMENSAT": "syscall", - "syscall.SYS_UTIMES": "syscall", - "syscall.SYS_UTRACE": "syscall", - "syscall.SYS_UUIDGEN": "syscall", - "syscall.SYS_VADVISE": "syscall", - "syscall.SYS_VFORK": "syscall", - "syscall.SYS_VHANGUP": "syscall", - "syscall.SYS_VM86": "syscall", - "syscall.SYS_VM86OLD": "syscall", - "syscall.SYS_VMSPLICE": "syscall", - "syscall.SYS_VM_PRESSURE_MONITOR": "syscall", - "syscall.SYS_VSERVER": "syscall", - "syscall.SYS_WAIT4": "syscall", - "syscall.SYS_WAIT4_NOCANCEL": "syscall", - "syscall.SYS_WAIT6": "syscall", - "syscall.SYS_WAITEVENT": "syscall", - "syscall.SYS_WAITID": "syscall", - "syscall.SYS_WAITID_NOCANCEL": "syscall", - "syscall.SYS_WAITPID": "syscall", - "syscall.SYS_WATCHEVENT": "syscall", - "syscall.SYS_WORKQ_KERNRETURN": "syscall", - "syscall.SYS_WORKQ_OPEN": "syscall", - "syscall.SYS_WRITE": "syscall", - "syscall.SYS_WRITEV": "syscall", - "syscall.SYS_WRITEV_NOCANCEL": "syscall", - "syscall.SYS_WRITE_NOCANCEL": "syscall", - "syscall.SYS_YIELD": "syscall", - "syscall.SYS__LLSEEK": "syscall", - "syscall.SYS__LWP_CONTINUE": "syscall", - "syscall.SYS__LWP_CREATE": "syscall", - "syscall.SYS__LWP_CTL": "syscall", - "syscall.SYS__LWP_DETACH": "syscall", - "syscall.SYS__LWP_EXIT": "syscall", - "syscall.SYS__LWP_GETNAME": "syscall", - "syscall.SYS__LWP_GETPRIVATE": "syscall", - "syscall.SYS__LWP_KILL": "syscall", - "syscall.SYS__LWP_PARK": "syscall", - "syscall.SYS__LWP_SELF": "syscall", - "syscall.SYS__LWP_SETNAME": "syscall", - "syscall.SYS__LWP_SETPRIVATE": "syscall", - "syscall.SYS__LWP_SUSPEND": "syscall", - "syscall.SYS__LWP_UNPARK": "syscall", - "syscall.SYS__LWP_UNPARK_ALL": "syscall", - "syscall.SYS__LWP_WAIT": "syscall", - "syscall.SYS__LWP_WAKEUP": "syscall", - "syscall.SYS__NEWSELECT": "syscall", - "syscall.SYS__PSET_BIND": "syscall", - "syscall.SYS__SCHED_GETAFFINITY": "syscall", - "syscall.SYS__SCHED_GETPARAM": "syscall", - "syscall.SYS__SCHED_SETAFFINITY": "syscall", - "syscall.SYS__SCHED_SETPARAM": "syscall", - "syscall.SYS__SYSCTL": "syscall", - "syscall.SYS__UMTX_LOCK": "syscall", - "syscall.SYS__UMTX_OP": "syscall", - "syscall.SYS__UMTX_UNLOCK": "syscall", - "syscall.SYS___ACL_ACLCHECK_FD": "syscall", - "syscall.SYS___ACL_ACLCHECK_FILE": "syscall", - "syscall.SYS___ACL_ACLCHECK_LINK": "syscall", - "syscall.SYS___ACL_DELETE_FD": "syscall", - "syscall.SYS___ACL_DELETE_FILE": "syscall", - "syscall.SYS___ACL_DELETE_LINK": "syscall", - "syscall.SYS___ACL_GET_FD": "syscall", - "syscall.SYS___ACL_GET_FILE": "syscall", - "syscall.SYS___ACL_GET_LINK": "syscall", - "syscall.SYS___ACL_SET_FD": "syscall", - "syscall.SYS___ACL_SET_FILE": "syscall", - "syscall.SYS___ACL_SET_LINK": "syscall", - "syscall.SYS___CLONE": "syscall", - "syscall.SYS___DISABLE_THREADSIGNAL": "syscall", - "syscall.SYS___GETCWD": "syscall", - "syscall.SYS___GETLOGIN": "syscall", - "syscall.SYS___GET_TCB": "syscall", - "syscall.SYS___MAC_EXECVE": "syscall", - "syscall.SYS___MAC_GETFSSTAT": "syscall", - "syscall.SYS___MAC_GET_FD": "syscall", - "syscall.SYS___MAC_GET_FILE": "syscall", - "syscall.SYS___MAC_GET_LCID": "syscall", - "syscall.SYS___MAC_GET_LCTX": "syscall", - "syscall.SYS___MAC_GET_LINK": "syscall", - "syscall.SYS___MAC_GET_MOUNT": "syscall", - "syscall.SYS___MAC_GET_PID": "syscall", - "syscall.SYS___MAC_GET_PROC": "syscall", - "syscall.SYS___MAC_MOUNT": "syscall", - "syscall.SYS___MAC_SET_FD": "syscall", - "syscall.SYS___MAC_SET_FILE": "syscall", - "syscall.SYS___MAC_SET_LCTX": "syscall", - "syscall.SYS___MAC_SET_LINK": "syscall", - "syscall.SYS___MAC_SET_PROC": "syscall", - "syscall.SYS___MAC_SYSCALL": "syscall", - "syscall.SYS___OLD_SEMWAIT_SIGNAL": "syscall", - "syscall.SYS___OLD_SEMWAIT_SIGNAL_NOCANCEL": "syscall", - "syscall.SYS___POSIX_CHOWN": "syscall", - "syscall.SYS___POSIX_FCHOWN": "syscall", - "syscall.SYS___POSIX_LCHOWN": "syscall", - "syscall.SYS___POSIX_RENAME": "syscall", - "syscall.SYS___PTHREAD_CANCELED": "syscall", - "syscall.SYS___PTHREAD_CHDIR": "syscall", - "syscall.SYS___PTHREAD_FCHDIR": "syscall", - "syscall.SYS___PTHREAD_KILL": "syscall", - "syscall.SYS___PTHREAD_MARKCANCEL": "syscall", - "syscall.SYS___PTHREAD_SIGMASK": "syscall", - "syscall.SYS___QUOTACTL": "syscall", - "syscall.SYS___SEMCTL": "syscall", - "syscall.SYS___SEMWAIT_SIGNAL": "syscall", - "syscall.SYS___SEMWAIT_SIGNAL_NOCANCEL": "syscall", - "syscall.SYS___SETLOGIN": "syscall", - "syscall.SYS___SETUGID": "syscall", - "syscall.SYS___SET_TCB": "syscall", - "syscall.SYS___SIGACTION_SIGTRAMP": "syscall", - "syscall.SYS___SIGTIMEDWAIT": "syscall", - "syscall.SYS___SIGWAIT": "syscall", - "syscall.SYS___SIGWAIT_NOCANCEL": "syscall", - "syscall.SYS___SYSCTL": "syscall", - "syscall.SYS___TFORK": "syscall", - "syscall.SYS___THREXIT": "syscall", - "syscall.SYS___THRSIGDIVERT": "syscall", - "syscall.SYS___THRSLEEP": "syscall", - "syscall.SYS___THRWAKEUP": "syscall", - "syscall.S_ARCH1": "syscall", - "syscall.S_ARCH2": "syscall", - "syscall.S_BLKSIZE": "syscall", - "syscall.S_IEXEC": "syscall", - "syscall.S_IFBLK": "syscall", - "syscall.S_IFCHR": "syscall", - "syscall.S_IFDIR": "syscall", - "syscall.S_IFIFO": "syscall", - "syscall.S_IFLNK": "syscall", - "syscall.S_IFMT": "syscall", - "syscall.S_IFREG": "syscall", - "syscall.S_IFSOCK": "syscall", - "syscall.S_IFWHT": "syscall", - "syscall.S_IREAD": "syscall", - "syscall.S_IRGRP": "syscall", - "syscall.S_IROTH": "syscall", - "syscall.S_IRUSR": "syscall", - "syscall.S_IRWXG": "syscall", - "syscall.S_IRWXO": "syscall", - "syscall.S_IRWXU": "syscall", - "syscall.S_ISGID": "syscall", - "syscall.S_ISTXT": "syscall", - "syscall.S_ISUID": "syscall", - "syscall.S_ISVTX": "syscall", - "syscall.S_IWGRP": "syscall", - "syscall.S_IWOTH": "syscall", - "syscall.S_IWRITE": "syscall", - "syscall.S_IWUSR": "syscall", - "syscall.S_IXGRP": "syscall", - "syscall.S_IXOTH": "syscall", - "syscall.S_IXUSR": "syscall", - "syscall.S_LOGIN_SET": "syscall", - "syscall.SecurityAttributes": "syscall", - "syscall.Seek": "syscall", - "syscall.Select": "syscall", - "syscall.Sendfile": "syscall", - "syscall.Sendmsg": "syscall", - "syscall.SendmsgN": "syscall", - "syscall.Sendto": "syscall", - "syscall.Servent": "syscall", - "syscall.SetBpf": "syscall", - "syscall.SetBpfBuflen": "syscall", - "syscall.SetBpfDatalink": "syscall", - "syscall.SetBpfHeadercmpl": "syscall", - "syscall.SetBpfImmediate": "syscall", - "syscall.SetBpfInterface": "syscall", - "syscall.SetBpfPromisc": "syscall", - "syscall.SetBpfTimeout": "syscall", - "syscall.SetCurrentDirectory": "syscall", - "syscall.SetEndOfFile": "syscall", - "syscall.SetEnvironmentVariable": "syscall", - "syscall.SetFileAttributes": "syscall", - "syscall.SetFileCompletionNotificationModes": "syscall", - "syscall.SetFilePointer": "syscall", - "syscall.SetFileTime": "syscall", - "syscall.SetHandleInformation": "syscall", - "syscall.SetKevent": "syscall", - "syscall.SetLsfPromisc": "syscall", - "syscall.SetNonblock": "syscall", - "syscall.Setdomainname": "syscall", - "syscall.Setegid": "syscall", - "syscall.Setenv": "syscall", - "syscall.Seteuid": "syscall", - "syscall.Setfsgid": "syscall", - "syscall.Setfsuid": "syscall", - "syscall.Setgid": "syscall", - "syscall.Setgroups": "syscall", - "syscall.Sethostname": "syscall", - "syscall.Setlogin": "syscall", - "syscall.Setpgid": "syscall", - "syscall.Setpriority": "syscall", - "syscall.Setprivexec": "syscall", - "syscall.Setregid": "syscall", - "syscall.Setresgid": "syscall", - "syscall.Setresuid": "syscall", - "syscall.Setreuid": "syscall", - "syscall.Setrlimit": "syscall", - "syscall.Setsid": "syscall", - "syscall.Setsockopt": "syscall", - "syscall.SetsockoptByte": "syscall", - "syscall.SetsockoptICMPv6Filter": "syscall", - "syscall.SetsockoptIPMreq": "syscall", - "syscall.SetsockoptIPMreqn": "syscall", - "syscall.SetsockoptIPv6Mreq": "syscall", - "syscall.SetsockoptInet4Addr": "syscall", - "syscall.SetsockoptInt": "syscall", - "syscall.SetsockoptLinger": "syscall", - "syscall.SetsockoptString": "syscall", - "syscall.SetsockoptTimeval": "syscall", - "syscall.Settimeofday": "syscall", - "syscall.Setuid": "syscall", - "syscall.Setxattr": "syscall", - "syscall.Shutdown": "syscall", - "syscall.SidTypeAlias": "syscall", - "syscall.SidTypeComputer": "syscall", - "syscall.SidTypeDeletedAccount": "syscall", - "syscall.SidTypeDomain": "syscall", - "syscall.SidTypeGroup": "syscall", - "syscall.SidTypeInvalid": "syscall", - "syscall.SidTypeLabel": "syscall", - "syscall.SidTypeUnknown": "syscall", - "syscall.SidTypeUser": "syscall", - "syscall.SidTypeWellKnownGroup": "syscall", - "syscall.Signal": "syscall", - "syscall.SizeofBpfHdr": "syscall", - "syscall.SizeofBpfInsn": "syscall", - "syscall.SizeofBpfProgram": "syscall", - "syscall.SizeofBpfStat": "syscall", - "syscall.SizeofBpfVersion": "syscall", - "syscall.SizeofBpfZbuf": "syscall", - "syscall.SizeofBpfZbufHeader": "syscall", - "syscall.SizeofCmsghdr": "syscall", - "syscall.SizeofICMPv6Filter": "syscall", - "syscall.SizeofIPMreq": "syscall", - "syscall.SizeofIPMreqn": "syscall", - "syscall.SizeofIPv6MTUInfo": "syscall", - "syscall.SizeofIPv6Mreq": "syscall", - "syscall.SizeofIfAddrmsg": "syscall", - "syscall.SizeofIfAnnounceMsghdr": "syscall", - "syscall.SizeofIfData": "syscall", - "syscall.SizeofIfInfomsg": "syscall", - "syscall.SizeofIfMsghdr": "syscall", - "syscall.SizeofIfaMsghdr": "syscall", - "syscall.SizeofIfmaMsghdr": "syscall", - "syscall.SizeofIfmaMsghdr2": "syscall", - "syscall.SizeofInet4Pktinfo": "syscall", - "syscall.SizeofInet6Pktinfo": "syscall", - "syscall.SizeofInotifyEvent": "syscall", - "syscall.SizeofLinger": "syscall", - "syscall.SizeofMsghdr": "syscall", - "syscall.SizeofNlAttr": "syscall", - "syscall.SizeofNlMsgerr": "syscall", - "syscall.SizeofNlMsghdr": "syscall", - "syscall.SizeofRtAttr": "syscall", - "syscall.SizeofRtGenmsg": "syscall", - "syscall.SizeofRtMetrics": "syscall", - "syscall.SizeofRtMsg": "syscall", - "syscall.SizeofRtMsghdr": "syscall", - "syscall.SizeofRtNexthop": "syscall", - "syscall.SizeofSockFilter": "syscall", - "syscall.SizeofSockFprog": "syscall", - "syscall.SizeofSockaddrAny": "syscall", - "syscall.SizeofSockaddrDatalink": "syscall", - "syscall.SizeofSockaddrInet4": "syscall", - "syscall.SizeofSockaddrInet6": "syscall", - "syscall.SizeofSockaddrLinklayer": "syscall", - "syscall.SizeofSockaddrNetlink": "syscall", - "syscall.SizeofSockaddrUnix": "syscall", - "syscall.SizeofTCPInfo": "syscall", - "syscall.SizeofUcred": "syscall", - "syscall.SlicePtrFromStrings": "syscall", - "syscall.SockFilter": "syscall", - "syscall.SockFprog": "syscall", - "syscall.SockaddrDatalink": "syscall", - "syscall.SockaddrGen": "syscall", - "syscall.SockaddrInet4": "syscall", - "syscall.SockaddrInet6": "syscall", - "syscall.SockaddrLinklayer": "syscall", - "syscall.SockaddrNetlink": "syscall", - "syscall.SockaddrUnix": "syscall", - "syscall.Socket": "syscall", - "syscall.SocketControlMessage": "syscall", - "syscall.SocketDisableIPv6": "syscall", - "syscall.Socketpair": "syscall", - "syscall.Splice": "syscall", - "syscall.StartProcess": "syscall", - "syscall.StartupInfo": "syscall", - "syscall.Stat": "syscall", - "syscall.Stat_t": "syscall", - "syscall.Statfs": "syscall", - "syscall.Statfs_t": "syscall", - "syscall.Stderr": "syscall", - "syscall.Stdin": "syscall", - "syscall.Stdout": "syscall", - "syscall.StringBytePtr": "syscall", - "syscall.StringByteSlice": "syscall", - "syscall.StringSlicePtr": "syscall", - "syscall.StringToSid": "syscall", - "syscall.StringToUTF16": "syscall", - "syscall.StringToUTF16Ptr": "syscall", - "syscall.Symlink": "syscall", - "syscall.Sync": "syscall", - "syscall.SyncFileRange": "syscall", - "syscall.SysProcAttr": "syscall", - "syscall.SysProcIDMap": "syscall", - "syscall.Syscall": "syscall", - "syscall.Syscall12": "syscall", - "syscall.Syscall15": "syscall", - "syscall.Syscall6": "syscall", - "syscall.Syscall9": "syscall", - "syscall.Sysctl": "syscall", - "syscall.SysctlUint32": "syscall", - "syscall.Sysctlnode": "syscall", - "syscall.Sysinfo": "syscall", - "syscall.Sysinfo_t": "syscall", - "syscall.Systemtime": "syscall", - "syscall.TCGETS": "syscall", - "syscall.TCIFLUSH": "syscall", - "syscall.TCIOFLUSH": "syscall", - "syscall.TCOFLUSH": "syscall", - "syscall.TCPInfo": "syscall", - "syscall.TCPKeepalive": "syscall", - "syscall.TCP_CA_NAME_MAX": "syscall", - "syscall.TCP_CONGCTL": "syscall", - "syscall.TCP_CONGESTION": "syscall", - "syscall.TCP_CONNECTIONTIMEOUT": "syscall", - "syscall.TCP_CORK": "syscall", - "syscall.TCP_DEFER_ACCEPT": "syscall", - "syscall.TCP_INFO": "syscall", - "syscall.TCP_KEEPALIVE": "syscall", - "syscall.TCP_KEEPCNT": "syscall", - "syscall.TCP_KEEPIDLE": "syscall", - "syscall.TCP_KEEPINIT": "syscall", - "syscall.TCP_KEEPINTVL": "syscall", - "syscall.TCP_LINGER2": "syscall", - "syscall.TCP_MAXBURST": "syscall", - "syscall.TCP_MAXHLEN": "syscall", - "syscall.TCP_MAXOLEN": "syscall", - "syscall.TCP_MAXSEG": "syscall", - "syscall.TCP_MAXWIN": "syscall", - "syscall.TCP_MAX_SACK": "syscall", - "syscall.TCP_MAX_WINSHIFT": "syscall", - "syscall.TCP_MD5SIG": "syscall", - "syscall.TCP_MD5SIG_MAXKEYLEN": "syscall", - "syscall.TCP_MINMSS": "syscall", - "syscall.TCP_MINMSSOVERLOAD": "syscall", - "syscall.TCP_MSS": "syscall", - "syscall.TCP_NODELAY": "syscall", - "syscall.TCP_NOOPT": "syscall", - "syscall.TCP_NOPUSH": "syscall", - "syscall.TCP_NSTATES": "syscall", - "syscall.TCP_QUICKACK": "syscall", - "syscall.TCP_RXT_CONNDROPTIME": "syscall", - "syscall.TCP_RXT_FINDROP": "syscall", - "syscall.TCP_SACK_ENABLE": "syscall", - "syscall.TCP_SYNCNT": "syscall", - "syscall.TCP_VENDOR": "syscall", - "syscall.TCP_WINDOW_CLAMP": "syscall", - "syscall.TCSAFLUSH": "syscall", - "syscall.TCSETS": "syscall", - "syscall.TF_DISCONNECT": "syscall", - "syscall.TF_REUSE_SOCKET": "syscall", - "syscall.TF_USE_DEFAULT_WORKER": "syscall", - "syscall.TF_USE_KERNEL_APC": "syscall", - "syscall.TF_USE_SYSTEM_THREAD": "syscall", - "syscall.TF_WRITE_BEHIND": "syscall", - "syscall.TH32CS_INHERIT": "syscall", - "syscall.TH32CS_SNAPALL": "syscall", - "syscall.TH32CS_SNAPHEAPLIST": "syscall", - "syscall.TH32CS_SNAPMODULE": "syscall", - "syscall.TH32CS_SNAPMODULE32": "syscall", - "syscall.TH32CS_SNAPPROCESS": "syscall", - "syscall.TH32CS_SNAPTHREAD": "syscall", - "syscall.TIME_ZONE_ID_DAYLIGHT": "syscall", - "syscall.TIME_ZONE_ID_STANDARD": "syscall", - "syscall.TIME_ZONE_ID_UNKNOWN": "syscall", - "syscall.TIOCCBRK": "syscall", - "syscall.TIOCCDTR": "syscall", - "syscall.TIOCCONS": "syscall", - "syscall.TIOCDCDTIMESTAMP": "syscall", - "syscall.TIOCDRAIN": "syscall", - "syscall.TIOCDSIMICROCODE": "syscall", - "syscall.TIOCEXCL": "syscall", - "syscall.TIOCEXT": "syscall", - "syscall.TIOCFLAG_CDTRCTS": "syscall", - "syscall.TIOCFLAG_CLOCAL": "syscall", - "syscall.TIOCFLAG_CRTSCTS": "syscall", - "syscall.TIOCFLAG_MDMBUF": "syscall", - "syscall.TIOCFLAG_PPS": "syscall", - "syscall.TIOCFLAG_SOFTCAR": "syscall", - "syscall.TIOCFLUSH": "syscall", - "syscall.TIOCGDEV": "syscall", - "syscall.TIOCGDRAINWAIT": "syscall", - "syscall.TIOCGETA": "syscall", - "syscall.TIOCGETD": "syscall", - "syscall.TIOCGFLAGS": "syscall", - "syscall.TIOCGICOUNT": "syscall", - "syscall.TIOCGLCKTRMIOS": "syscall", - "syscall.TIOCGLINED": "syscall", - "syscall.TIOCGPGRP": "syscall", - "syscall.TIOCGPTN": "syscall", - "syscall.TIOCGQSIZE": "syscall", - "syscall.TIOCGRANTPT": "syscall", - "syscall.TIOCGRS485": "syscall", - "syscall.TIOCGSERIAL": "syscall", - "syscall.TIOCGSID": "syscall", - "syscall.TIOCGSIZE": "syscall", - "syscall.TIOCGSOFTCAR": "syscall", - "syscall.TIOCGTSTAMP": "syscall", - "syscall.TIOCGWINSZ": "syscall", - "syscall.TIOCINQ": "syscall", - "syscall.TIOCIXOFF": "syscall", - "syscall.TIOCIXON": "syscall", - "syscall.TIOCLINUX": "syscall", - "syscall.TIOCMBIC": "syscall", - "syscall.TIOCMBIS": "syscall", - "syscall.TIOCMGDTRWAIT": "syscall", - "syscall.TIOCMGET": "syscall", - "syscall.TIOCMIWAIT": "syscall", - "syscall.TIOCMODG": "syscall", - "syscall.TIOCMODS": "syscall", - "syscall.TIOCMSDTRWAIT": "syscall", - "syscall.TIOCMSET": "syscall", - "syscall.TIOCM_CAR": "syscall", - "syscall.TIOCM_CD": "syscall", - "syscall.TIOCM_CTS": "syscall", - "syscall.TIOCM_DCD": "syscall", - "syscall.TIOCM_DSR": "syscall", - "syscall.TIOCM_DTR": "syscall", - "syscall.TIOCM_LE": "syscall", - "syscall.TIOCM_RI": "syscall", - "syscall.TIOCM_RNG": "syscall", - "syscall.TIOCM_RTS": "syscall", - "syscall.TIOCM_SR": "syscall", - "syscall.TIOCM_ST": "syscall", - "syscall.TIOCNOTTY": "syscall", - "syscall.TIOCNXCL": "syscall", - "syscall.TIOCOUTQ": "syscall", - "syscall.TIOCPKT": "syscall", - "syscall.TIOCPKT_DATA": "syscall", - "syscall.TIOCPKT_DOSTOP": "syscall", - "syscall.TIOCPKT_FLUSHREAD": "syscall", - "syscall.TIOCPKT_FLUSHWRITE": "syscall", - "syscall.TIOCPKT_IOCTL": "syscall", - "syscall.TIOCPKT_NOSTOP": "syscall", - "syscall.TIOCPKT_START": "syscall", - "syscall.TIOCPKT_STOP": "syscall", - "syscall.TIOCPTMASTER": "syscall", - "syscall.TIOCPTMGET": "syscall", - "syscall.TIOCPTSNAME": "syscall", - "syscall.TIOCPTYGNAME": "syscall", - "syscall.TIOCPTYGRANT": "syscall", - "syscall.TIOCPTYUNLK": "syscall", - "syscall.TIOCRCVFRAME": "syscall", - "syscall.TIOCREMOTE": "syscall", - "syscall.TIOCSBRK": "syscall", - "syscall.TIOCSCONS": "syscall", - "syscall.TIOCSCTTY": "syscall", - "syscall.TIOCSDRAINWAIT": "syscall", - "syscall.TIOCSDTR": "syscall", - "syscall.TIOCSERCONFIG": "syscall", - "syscall.TIOCSERGETLSR": "syscall", - "syscall.TIOCSERGETMULTI": "syscall", - "syscall.TIOCSERGSTRUCT": "syscall", - "syscall.TIOCSERGWILD": "syscall", - "syscall.TIOCSERSETMULTI": "syscall", - "syscall.TIOCSERSWILD": "syscall", - "syscall.TIOCSER_TEMT": "syscall", - "syscall.TIOCSETA": "syscall", - "syscall.TIOCSETAF": "syscall", - "syscall.TIOCSETAW": "syscall", - "syscall.TIOCSETD": "syscall", - "syscall.TIOCSFLAGS": "syscall", - "syscall.TIOCSIG": "syscall", - "syscall.TIOCSLCKTRMIOS": "syscall", - "syscall.TIOCSLINED": "syscall", - "syscall.TIOCSPGRP": "syscall", - "syscall.TIOCSPTLCK": "syscall", - "syscall.TIOCSQSIZE": "syscall", - "syscall.TIOCSRS485": "syscall", - "syscall.TIOCSSERIAL": "syscall", - "syscall.TIOCSSIZE": "syscall", - "syscall.TIOCSSOFTCAR": "syscall", - "syscall.TIOCSTART": "syscall", - "syscall.TIOCSTAT": "syscall", - "syscall.TIOCSTI": "syscall", - "syscall.TIOCSTOP": "syscall", - "syscall.TIOCSTSTAMP": "syscall", - "syscall.TIOCSWINSZ": "syscall", - "syscall.TIOCTIMESTAMP": "syscall", - "syscall.TIOCUCNTL": "syscall", - "syscall.TIOCVHANGUP": "syscall", - "syscall.TIOCXMTFRAME": "syscall", - "syscall.TOKEN_ADJUST_DEFAULT": "syscall", - "syscall.TOKEN_ADJUST_GROUPS": "syscall", - "syscall.TOKEN_ADJUST_PRIVILEGES": "syscall", - "syscall.TOKEN_ALL_ACCESS": "syscall", - "syscall.TOKEN_ASSIGN_PRIMARY": "syscall", - "syscall.TOKEN_DUPLICATE": "syscall", - "syscall.TOKEN_EXECUTE": "syscall", - "syscall.TOKEN_IMPERSONATE": "syscall", - "syscall.TOKEN_QUERY": "syscall", - "syscall.TOKEN_QUERY_SOURCE": "syscall", - "syscall.TOKEN_READ": "syscall", - "syscall.TOKEN_WRITE": "syscall", - "syscall.TOSTOP": "syscall", - "syscall.TRUNCATE_EXISTING": "syscall", - "syscall.TUNATTACHFILTER": "syscall", - "syscall.TUNDETACHFILTER": "syscall", - "syscall.TUNGETFEATURES": "syscall", - "syscall.TUNGETIFF": "syscall", - "syscall.TUNGETSNDBUF": "syscall", - "syscall.TUNGETVNETHDRSZ": "syscall", - "syscall.TUNSETDEBUG": "syscall", - "syscall.TUNSETGROUP": "syscall", - "syscall.TUNSETIFF": "syscall", - "syscall.TUNSETLINK": "syscall", - "syscall.TUNSETNOCSUM": "syscall", - "syscall.TUNSETOFFLOAD": "syscall", - "syscall.TUNSETOWNER": "syscall", - "syscall.TUNSETPERSIST": "syscall", - "syscall.TUNSETSNDBUF": "syscall", - "syscall.TUNSETTXFILTER": "syscall", - "syscall.TUNSETVNETHDRSZ": "syscall", - "syscall.Tee": "syscall", - "syscall.TerminateProcess": "syscall", - "syscall.Termios": "syscall", - "syscall.Tgkill": "syscall", - "syscall.Time": "syscall", - "syscall.Time_t": "syscall", - "syscall.Times": "syscall", - "syscall.Timespec": "syscall", - "syscall.TimespecToNsec": "syscall", - "syscall.Timeval": "syscall", - "syscall.Timeval32": "syscall", - "syscall.TimevalToNsec": "syscall", - "syscall.Timex": "syscall", - "syscall.Timezoneinformation": "syscall", - "syscall.Tms": "syscall", - "syscall.Token": "syscall", - "syscall.TokenAccessInformation": "syscall", - "syscall.TokenAuditPolicy": "syscall", - "syscall.TokenDefaultDacl": "syscall", - "syscall.TokenElevation": "syscall", - "syscall.TokenElevationType": "syscall", - "syscall.TokenGroups": "syscall", - "syscall.TokenGroupsAndPrivileges": "syscall", - "syscall.TokenHasRestrictions": "syscall", - "syscall.TokenImpersonationLevel": "syscall", - "syscall.TokenIntegrityLevel": "syscall", - "syscall.TokenLinkedToken": "syscall", - "syscall.TokenLogonSid": "syscall", - "syscall.TokenMandatoryPolicy": "syscall", - "syscall.TokenOrigin": "syscall", - "syscall.TokenOwner": "syscall", - "syscall.TokenPrimaryGroup": "syscall", - "syscall.TokenPrivileges": "syscall", - "syscall.TokenRestrictedSids": "syscall", - "syscall.TokenSandBoxInert": "syscall", - "syscall.TokenSessionId": "syscall", - "syscall.TokenSessionReference": "syscall", - "syscall.TokenSource": "syscall", - "syscall.TokenStatistics": "syscall", - "syscall.TokenType": "syscall", - "syscall.TokenUIAccess": "syscall", - "syscall.TokenUser": "syscall", - "syscall.TokenVirtualizationAllowed": "syscall", - "syscall.TokenVirtualizationEnabled": "syscall", - "syscall.Tokenprimarygroup": "syscall", - "syscall.Tokenuser": "syscall", - "syscall.TranslateAccountName": "syscall", - "syscall.TranslateName": "syscall", - "syscall.TransmitFile": "syscall", - "syscall.TransmitFileBuffers": "syscall", - "syscall.Truncate": "syscall", - "syscall.USAGE_MATCH_TYPE_AND": "syscall", - "syscall.USAGE_MATCH_TYPE_OR": "syscall", - "syscall.UTF16FromString": "syscall", - "syscall.UTF16PtrFromString": "syscall", - "syscall.UTF16ToString": "syscall", - "syscall.Ucred": "syscall", - "syscall.Umask": "syscall", - "syscall.Uname": "syscall", - "syscall.Undelete": "syscall", - "syscall.UnixCredentials": "syscall", - "syscall.UnixRights": "syscall", - "syscall.Unlink": "syscall", - "syscall.Unlinkat": "syscall", - "syscall.UnmapViewOfFile": "syscall", - "syscall.Unmount": "syscall", - "syscall.Unsetenv": "syscall", - "syscall.Unshare": "syscall", - "syscall.UserInfo10": "syscall", - "syscall.Ustat": "syscall", - "syscall.Ustat_t": "syscall", - "syscall.Utimbuf": "syscall", - "syscall.Utime": "syscall", - "syscall.Utimes": "syscall", - "syscall.UtimesNano": "syscall", - "syscall.Utsname": "syscall", - "syscall.VDISCARD": "syscall", - "syscall.VDSUSP": "syscall", - "syscall.VEOF": "syscall", - "syscall.VEOL": "syscall", - "syscall.VEOL2": "syscall", - "syscall.VERASE": "syscall", - "syscall.VERASE2": "syscall", - "syscall.VINTR": "syscall", - "syscall.VKILL": "syscall", - "syscall.VLNEXT": "syscall", - "syscall.VMIN": "syscall", - "syscall.VQUIT": "syscall", - "syscall.VREPRINT": "syscall", - "syscall.VSTART": "syscall", - "syscall.VSTATUS": "syscall", - "syscall.VSTOP": "syscall", - "syscall.VSUSP": "syscall", - "syscall.VSWTC": "syscall", - "syscall.VT0": "syscall", - "syscall.VT1": "syscall", - "syscall.VTDLY": "syscall", - "syscall.VTIME": "syscall", - "syscall.VWERASE": "syscall", - "syscall.VirtualLock": "syscall", - "syscall.VirtualUnlock": "syscall", - "syscall.WAIT_ABANDONED": "syscall", - "syscall.WAIT_FAILED": "syscall", - "syscall.WAIT_OBJECT_0": "syscall", - "syscall.WAIT_TIMEOUT": "syscall", - "syscall.WALL": "syscall", - "syscall.WALLSIG": "syscall", - "syscall.WALTSIG": "syscall", - "syscall.WCLONE": "syscall", - "syscall.WCONTINUED": "syscall", - "syscall.WCOREFLAG": "syscall", - "syscall.WEXITED": "syscall", - "syscall.WLINUXCLONE": "syscall", - "syscall.WNOHANG": "syscall", - "syscall.WNOTHREAD": "syscall", - "syscall.WNOWAIT": "syscall", - "syscall.WNOZOMBIE": "syscall", - "syscall.WOPTSCHECKED": "syscall", - "syscall.WORDSIZE": "syscall", - "syscall.WSABuf": "syscall", - "syscall.WSACleanup": "syscall", - "syscall.WSADESCRIPTION_LEN": "syscall", - "syscall.WSAData": "syscall", - "syscall.WSAEACCES": "syscall", - "syscall.WSAECONNRESET": "syscall", - "syscall.WSAEnumProtocols": "syscall", - "syscall.WSAID_CONNECTEX": "syscall", - "syscall.WSAIoctl": "syscall", - "syscall.WSAPROTOCOL_LEN": "syscall", - "syscall.WSAProtocolChain": "syscall", - "syscall.WSAProtocolInfo": "syscall", - "syscall.WSARecv": "syscall", - "syscall.WSARecvFrom": "syscall", - "syscall.WSASYS_STATUS_LEN": "syscall", - "syscall.WSASend": "syscall", - "syscall.WSASendTo": "syscall", - "syscall.WSASendto": "syscall", - "syscall.WSAStartup": "syscall", - "syscall.WSTOPPED": "syscall", - "syscall.WTRAPPED": "syscall", - "syscall.WUNTRACED": "syscall", - "syscall.Wait4": "syscall", - "syscall.WaitForSingleObject": "syscall", - "syscall.WaitStatus": "syscall", - "syscall.Win32FileAttributeData": "syscall", - "syscall.Win32finddata": "syscall", - "syscall.Write": "syscall", - "syscall.WriteConsole": "syscall", - "syscall.WriteFile": "syscall", - "syscall.X509_ASN_ENCODING": "syscall", - "syscall.XCASE": "syscall", - "syscall.XP1_CONNECTIONLESS": "syscall", - "syscall.XP1_CONNECT_DATA": "syscall", - "syscall.XP1_DISCONNECT_DATA": "syscall", - "syscall.XP1_EXPEDITED_DATA": "syscall", - "syscall.XP1_GRACEFUL_CLOSE": "syscall", - "syscall.XP1_GUARANTEED_DELIVERY": "syscall", - "syscall.XP1_GUARANTEED_ORDER": "syscall", - "syscall.XP1_IFS_HANDLES": "syscall", - "syscall.XP1_MESSAGE_ORIENTED": "syscall", - "syscall.XP1_MULTIPOINT_CONTROL_PLANE": "syscall", - "syscall.XP1_MULTIPOINT_DATA_PLANE": "syscall", - "syscall.XP1_PARTIAL_MESSAGE": "syscall", - "syscall.XP1_PSEUDO_STREAM": "syscall", - "syscall.XP1_QOS_SUPPORTED": "syscall", - "syscall.XP1_SAN_SUPPORT_SDP": "syscall", - "syscall.XP1_SUPPORT_BROADCAST": "syscall", - "syscall.XP1_SUPPORT_MULTIPOINT": "syscall", - "syscall.XP1_UNI_RECV": "syscall", - "syscall.XP1_UNI_SEND": "syscall", - "syslog.Dial": "log/syslog", - "syslog.LOG_ALERT": "log/syslog", - "syslog.LOG_AUTH": "log/syslog", - "syslog.LOG_AUTHPRIV": "log/syslog", - "syslog.LOG_CRIT": "log/syslog", - "syslog.LOG_CRON": "log/syslog", - "syslog.LOG_DAEMON": "log/syslog", - "syslog.LOG_DEBUG": "log/syslog", - "syslog.LOG_EMERG": "log/syslog", - "syslog.LOG_ERR": "log/syslog", - "syslog.LOG_FTP": "log/syslog", - "syslog.LOG_INFO": "log/syslog", - "syslog.LOG_KERN": "log/syslog", - "syslog.LOG_LOCAL0": "log/syslog", - "syslog.LOG_LOCAL1": "log/syslog", - "syslog.LOG_LOCAL2": "log/syslog", - "syslog.LOG_LOCAL3": "log/syslog", - "syslog.LOG_LOCAL4": "log/syslog", - "syslog.LOG_LOCAL5": "log/syslog", - "syslog.LOG_LOCAL6": "log/syslog", - "syslog.LOG_LOCAL7": "log/syslog", - "syslog.LOG_LPR": "log/syslog", - "syslog.LOG_MAIL": "log/syslog", - "syslog.LOG_NEWS": "log/syslog", - "syslog.LOG_NOTICE": "log/syslog", - "syslog.LOG_SYSLOG": "log/syslog", - "syslog.LOG_USER": "log/syslog", - "syslog.LOG_UUCP": "log/syslog", - "syslog.LOG_WARNING": "log/syslog", - "syslog.New": "log/syslog", - "syslog.NewLogger": "log/syslog", - "syslog.Priority": "log/syslog", - "syslog.Writer": "log/syslog", - "tabwriter.AlignRight": "text/tabwriter", - "tabwriter.Debug": "text/tabwriter", - "tabwriter.DiscardEmptyColumns": "text/tabwriter", - "tabwriter.Escape": "text/tabwriter", - "tabwriter.FilterHTML": "text/tabwriter", - "tabwriter.NewWriter": "text/tabwriter", - "tabwriter.StripEscape": "text/tabwriter", - "tabwriter.TabIndent": "text/tabwriter", - "tabwriter.Writer": "text/tabwriter", - "tar.ErrFieldTooLong": "archive/tar", - "tar.ErrHeader": "archive/tar", - "tar.ErrWriteAfterClose": "archive/tar", - "tar.ErrWriteTooLong": "archive/tar", - "tar.FileInfoHeader": "archive/tar", - "tar.Header": "archive/tar", - "tar.NewReader": "archive/tar", - "tar.NewWriter": "archive/tar", - "tar.Reader": "archive/tar", - "tar.TypeBlock": "archive/tar", - "tar.TypeChar": "archive/tar", - "tar.TypeCont": "archive/tar", - "tar.TypeDir": "archive/tar", - "tar.TypeFifo": "archive/tar", - "tar.TypeGNULongLink": "archive/tar", - "tar.TypeGNULongName": "archive/tar", - "tar.TypeGNUSparse": "archive/tar", - "tar.TypeLink": "archive/tar", - "tar.TypeReg": "archive/tar", - "tar.TypeRegA": "archive/tar", - "tar.TypeSymlink": "archive/tar", - "tar.TypeXGlobalHeader": "archive/tar", - "tar.TypeXHeader": "archive/tar", - "tar.Writer": "archive/tar", - "template.CSS": "html/template", - "template.ErrAmbigContext": "html/template", - "template.ErrBadHTML": "html/template", - "template.ErrBranchEnd": "html/template", - "template.ErrEndContext": "html/template", - "template.ErrNoSuchTemplate": "html/template", - "template.ErrOutputContext": "html/template", - "template.ErrPartialCharset": "html/template", - "template.ErrPartialEscape": "html/template", - "template.ErrRangeLoopReentry": "html/template", - "template.ErrSlashAmbig": "html/template", - "template.Error": "html/template", - "template.ErrorCode": "html/template", - "template.ExecError": "text/template", - // "template.FuncMap" is ambiguous - "template.HTML": "html/template", - "template.HTMLAttr": "html/template", - // "template.HTMLEscape" is ambiguous - // "template.HTMLEscapeString" is ambiguous - // "template.HTMLEscaper" is ambiguous - // "template.IsTrue" is ambiguous - "template.JS": "html/template", - // "template.JSEscape" is ambiguous - // "template.JSEscapeString" is ambiguous - // "template.JSEscaper" is ambiguous - "template.JSStr": "html/template", - // "template.Must" is ambiguous - // "template.New" is ambiguous - "template.OK": "html/template", - // "template.ParseFiles" is ambiguous - // "template.ParseGlob" is ambiguous - // "template.Template" is ambiguous - "template.URL": "html/template", - // "template.URLQueryEscaper" is ambiguous - "testing.AllocsPerRun": "testing", - "testing.B": "testing", - "testing.Benchmark": "testing", - "testing.BenchmarkResult": "testing", - "testing.Cover": "testing", - "testing.CoverBlock": "testing", - "testing.Coverage": "testing", - "testing.InternalBenchmark": "testing", - "testing.InternalExample": "testing", - "testing.InternalTest": "testing", - "testing.M": "testing", - "testing.Main": "testing", - "testing.MainStart": "testing", - "testing.PB": "testing", - "testing.RegisterCover": "testing", - "testing.RunBenchmarks": "testing", - "testing.RunExamples": "testing", - "testing.RunTests": "testing", - "testing.Short": "testing", - "testing.T": "testing", - "testing.Verbose": "testing", - "textproto.CanonicalMIMEHeaderKey": "net/textproto", - "textproto.Conn": "net/textproto", - "textproto.Dial": "net/textproto", - "textproto.Error": "net/textproto", - "textproto.MIMEHeader": "net/textproto", - "textproto.NewConn": "net/textproto", - "textproto.NewReader": "net/textproto", - "textproto.NewWriter": "net/textproto", - "textproto.Pipeline": "net/textproto", - "textproto.ProtocolError": "net/textproto", - "textproto.Reader": "net/textproto", - "textproto.TrimBytes": "net/textproto", - "textproto.TrimString": "net/textproto", - "textproto.Writer": "net/textproto", - "time.ANSIC": "time", - "time.After": "time", - "time.AfterFunc": "time", - "time.April": "time", - "time.August": "time", - "time.Date": "time", - "time.December": "time", - "time.Duration": "time", - "time.February": "time", - "time.FixedZone": "time", - "time.Friday": "time", - "time.Hour": "time", - "time.January": "time", - "time.July": "time", - "time.June": "time", - "time.Kitchen": "time", - "time.LoadLocation": "time", - "time.Local": "time", - "time.Location": "time", - "time.March": "time", - "time.May": "time", - "time.Microsecond": "time", - "time.Millisecond": "time", - "time.Minute": "time", - "time.Monday": "time", - "time.Month": "time", - "time.Nanosecond": "time", - "time.NewTicker": "time", - "time.NewTimer": "time", - "time.November": "time", - "time.Now": "time", - "time.October": "time", - "time.Parse": "time", - "time.ParseDuration": "time", - "time.ParseError": "time", - "time.ParseInLocation": "time", - "time.RFC1123": "time", - "time.RFC1123Z": "time", - "time.RFC3339": "time", - "time.RFC3339Nano": "time", - "time.RFC822": "time", - "time.RFC822Z": "time", - "time.RFC850": "time", - "time.RubyDate": "time", - "time.Saturday": "time", - "time.Second": "time", - "time.September": "time", - "time.Since": "time", - "time.Sleep": "time", - "time.Stamp": "time", - "time.StampMicro": "time", - "time.StampMilli": "time", - "time.StampNano": "time", - "time.Sunday": "time", - "time.Thursday": "time", - "time.Tick": "time", - "time.Ticker": "time", - "time.Time": "time", - "time.Timer": "time", - "time.Tuesday": "time", - "time.UTC": "time", - "time.Unix": "time", - "time.UnixDate": "time", - "time.Wednesday": "time", - "time.Weekday": "time", - "tls.Certificate": "crypto/tls", - "tls.Client": "crypto/tls", - "tls.ClientAuthType": "crypto/tls", - "tls.ClientHelloInfo": "crypto/tls", - "tls.ClientSessionCache": "crypto/tls", - "tls.ClientSessionState": "crypto/tls", - "tls.Config": "crypto/tls", - "tls.Conn": "crypto/tls", - "tls.ConnectionState": "crypto/tls", - "tls.CurveID": "crypto/tls", - "tls.CurveP256": "crypto/tls", - "tls.CurveP384": "crypto/tls", - "tls.CurveP521": "crypto/tls", - "tls.Dial": "crypto/tls", - "tls.DialWithDialer": "crypto/tls", - "tls.Listen": "crypto/tls", - "tls.LoadX509KeyPair": "crypto/tls", - "tls.NewLRUClientSessionCache": "crypto/tls", - "tls.NewListener": "crypto/tls", - "tls.NoClientCert": "crypto/tls", - "tls.RecordHeaderError": "crypto/tls", - "tls.RenegotiateFreelyAsClient": "crypto/tls", - "tls.RenegotiateNever": "crypto/tls", - "tls.RenegotiateOnceAsClient": "crypto/tls", - "tls.RenegotiationSupport": "crypto/tls", - "tls.RequestClientCert": "crypto/tls", - "tls.RequireAndVerifyClientCert": "crypto/tls", - "tls.RequireAnyClientCert": "crypto/tls", - "tls.Server": "crypto/tls", - "tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA": "crypto/tls", - "tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": "crypto/tls", - "tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA": "crypto/tls", - "tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": "crypto/tls", - "tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA": "crypto/tls", - "tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA": "crypto/tls", - "tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA": "crypto/tls", - "tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": "crypto/tls", - "tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA": "crypto/tls", - "tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": "crypto/tls", - "tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA": "crypto/tls", - "tls.TLS_FALLBACK_SCSV": "crypto/tls", - "tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA": "crypto/tls", - "tls.TLS_RSA_WITH_AES_128_CBC_SHA": "crypto/tls", - "tls.TLS_RSA_WITH_AES_128_GCM_SHA256": "crypto/tls", - "tls.TLS_RSA_WITH_AES_256_CBC_SHA": "crypto/tls", - "tls.TLS_RSA_WITH_AES_256_GCM_SHA384": "crypto/tls", - "tls.TLS_RSA_WITH_RC4_128_SHA": "crypto/tls", - "tls.VerifyClientCertIfGiven": "crypto/tls", - "tls.VersionSSL30": "crypto/tls", - "tls.VersionTLS10": "crypto/tls", - "tls.VersionTLS11": "crypto/tls", - "tls.VersionTLS12": "crypto/tls", - "tls.X509KeyPair": "crypto/tls", - "token.ADD": "go/token", - "token.ADD_ASSIGN": "go/token", - "token.AND": "go/token", - "token.AND_ASSIGN": "go/token", - "token.AND_NOT": "go/token", - "token.AND_NOT_ASSIGN": "go/token", - "token.ARROW": "go/token", - "token.ASSIGN": "go/token", - "token.BREAK": "go/token", - "token.CASE": "go/token", - "token.CHAN": "go/token", - "token.CHAR": "go/token", - "token.COLON": "go/token", - "token.COMMA": "go/token", - "token.COMMENT": "go/token", - "token.CONST": "go/token", - "token.CONTINUE": "go/token", - "token.DEC": "go/token", - "token.DEFAULT": "go/token", - "token.DEFER": "go/token", - "token.DEFINE": "go/token", - "token.ELLIPSIS": "go/token", - "token.ELSE": "go/token", - "token.EOF": "go/token", - "token.EQL": "go/token", - "token.FALLTHROUGH": "go/token", - "token.FLOAT": "go/token", - "token.FOR": "go/token", - "token.FUNC": "go/token", - "token.File": "go/token", - "token.FileSet": "go/token", - "token.GEQ": "go/token", - "token.GO": "go/token", - "token.GOTO": "go/token", - "token.GTR": "go/token", - "token.HighestPrec": "go/token", - "token.IDENT": "go/token", - "token.IF": "go/token", - "token.ILLEGAL": "go/token", - "token.IMAG": "go/token", - "token.IMPORT": "go/token", - "token.INC": "go/token", - "token.INT": "go/token", - "token.INTERFACE": "go/token", - "token.LAND": "go/token", - "token.LBRACE": "go/token", - "token.LBRACK": "go/token", - "token.LEQ": "go/token", - "token.LOR": "go/token", - "token.LPAREN": "go/token", - "token.LSS": "go/token", - "token.Lookup": "go/token", - "token.LowestPrec": "go/token", - "token.MAP": "go/token", - "token.MUL": "go/token", - "token.MUL_ASSIGN": "go/token", - "token.NEQ": "go/token", - "token.NOT": "go/token", - "token.NewFileSet": "go/token", - "token.NoPos": "go/token", - "token.OR": "go/token", - "token.OR_ASSIGN": "go/token", - "token.PACKAGE": "go/token", - "token.PERIOD": "go/token", - "token.Pos": "go/token", - "token.Position": "go/token", - "token.QUO": "go/token", - "token.QUO_ASSIGN": "go/token", - "token.RANGE": "go/token", - "token.RBRACE": "go/token", - "token.RBRACK": "go/token", - "token.REM": "go/token", - "token.REM_ASSIGN": "go/token", - "token.RETURN": "go/token", - "token.RPAREN": "go/token", - "token.SELECT": "go/token", - "token.SEMICOLON": "go/token", - "token.SHL": "go/token", - "token.SHL_ASSIGN": "go/token", - "token.SHR": "go/token", - "token.SHR_ASSIGN": "go/token", - "token.STRING": "go/token", - "token.STRUCT": "go/token", - "token.SUB": "go/token", - "token.SUB_ASSIGN": "go/token", - "token.SWITCH": "go/token", - "token.TYPE": "go/token", - "token.Token": "go/token", - "token.UnaryPrec": "go/token", - "token.VAR": "go/token", - "token.XOR": "go/token", - "token.XOR_ASSIGN": "go/token", - "trace.Start": "runtime/trace", - "trace.Stop": "runtime/trace", - "types.Array": "go/types", - "types.AssertableTo": "go/types", - "types.AssignableTo": "go/types", - "types.Basic": "go/types", - "types.BasicInfo": "go/types", - "types.BasicKind": "go/types", - "types.Bool": "go/types", - "types.Builtin": "go/types", - "types.Byte": "go/types", - "types.Chan": "go/types", - "types.ChanDir": "go/types", - "types.Checker": "go/types", - "types.Comparable": "go/types", - "types.Complex128": "go/types", - "types.Complex64": "go/types", - "types.Config": "go/types", - "types.Const": "go/types", - "types.ConvertibleTo": "go/types", - "types.DefPredeclaredTestFuncs": "go/types", - "types.Error": "go/types", - "types.Eval": "go/types", - "types.ExprString": "go/types", - "types.FieldVal": "go/types", - "types.Float32": "go/types", - "types.Float64": "go/types", - "types.Func": "go/types", - "types.Id": "go/types", - "types.Identical": "go/types", - "types.Implements": "go/types", - "types.ImportMode": "go/types", - "types.Importer": "go/types", - "types.ImporterFrom": "go/types", - "types.Info": "go/types", - "types.Initializer": "go/types", - "types.Int": "go/types", - "types.Int16": "go/types", - "types.Int32": "go/types", - "types.Int64": "go/types", - "types.Int8": "go/types", - "types.Interface": "go/types", - "types.Invalid": "go/types", - "types.IsBoolean": "go/types", - "types.IsComplex": "go/types", - "types.IsConstType": "go/types", - "types.IsFloat": "go/types", - "types.IsInteger": "go/types", - "types.IsInterface": "go/types", - "types.IsNumeric": "go/types", - "types.IsOrdered": "go/types", - "types.IsString": "go/types", - "types.IsUnsigned": "go/types", - "types.IsUntyped": "go/types", - "types.Label": "go/types", - "types.LookupFieldOrMethod": "go/types", - "types.Map": "go/types", - "types.MethodExpr": "go/types", - "types.MethodSet": "go/types", - "types.MethodVal": "go/types", - "types.MissingMethod": "go/types", - "types.Named": "go/types", - "types.NewArray": "go/types", - "types.NewChan": "go/types", - "types.NewChecker": "go/types", - "types.NewConst": "go/types", - "types.NewField": "go/types", - "types.NewFunc": "go/types", - "types.NewInterface": "go/types", - "types.NewLabel": "go/types", - "types.NewMap": "go/types", - "types.NewMethodSet": "go/types", - "types.NewNamed": "go/types", - "types.NewPackage": "go/types", - "types.NewParam": "go/types", - "types.NewPkgName": "go/types", - "types.NewPointer": "go/types", - "types.NewScope": "go/types", - "types.NewSignature": "go/types", - "types.NewSlice": "go/types", - "types.NewStruct": "go/types", - "types.NewTuple": "go/types", - "types.NewTypeName": "go/types", - "types.NewVar": "go/types", - "types.Nil": "go/types", - "types.ObjectString": "go/types", - "types.Package": "go/types", - "types.PkgName": "go/types", - "types.Pointer": "go/types", - "types.Qualifier": "go/types", - "types.RecvOnly": "go/types", - "types.RelativeTo": "go/types", - "types.Rune": "go/types", - "types.Scope": "go/types", - "types.Selection": "go/types", - "types.SelectionKind": "go/types", - "types.SelectionString": "go/types", - "types.SendOnly": "go/types", - "types.SendRecv": "go/types", - "types.Signature": "go/types", - "types.Sizes": "go/types", - "types.Slice": "go/types", - "types.StdSizes": "go/types", - "types.String": "go/types", - "types.Struct": "go/types", - "types.Tuple": "go/types", - "types.Typ": "go/types", - "types.Type": "go/types", - "types.TypeAndValue": "go/types", - "types.TypeName": "go/types", - "types.TypeString": "go/types", - "types.Uint": "go/types", - "types.Uint16": "go/types", - "types.Uint32": "go/types", - "types.Uint64": "go/types", - "types.Uint8": "go/types", - "types.Uintptr": "go/types", - "types.Universe": "go/types", - "types.Unsafe": "go/types", - "types.UnsafePointer": "go/types", - "types.UntypedBool": "go/types", - "types.UntypedComplex": "go/types", - "types.UntypedFloat": "go/types", - "types.UntypedInt": "go/types", - "types.UntypedNil": "go/types", - "types.UntypedRune": "go/types", - "types.UntypedString": "go/types", - "types.Var": "go/types", - "types.WriteExpr": "go/types", - "types.WriteSignature": "go/types", - "types.WriteType": "go/types", - "unicode.ASCII_Hex_Digit": "unicode", - "unicode.Adlam": "unicode", - "unicode.Ahom": "unicode", - "unicode.Anatolian_Hieroglyphs": "unicode", - "unicode.Arabic": "unicode", - "unicode.Armenian": "unicode", - "unicode.Avestan": "unicode", - "unicode.AzeriCase": "unicode", - "unicode.Balinese": "unicode", - "unicode.Bamum": "unicode", - "unicode.Bassa_Vah": "unicode", - "unicode.Batak": "unicode", - "unicode.Bengali": "unicode", - "unicode.Bhaiksuki": "unicode", - "unicode.Bidi_Control": "unicode", - "unicode.Bopomofo": "unicode", - "unicode.Brahmi": "unicode", - "unicode.Braille": "unicode", - "unicode.Buginese": "unicode", - "unicode.Buhid": "unicode", - "unicode.C": "unicode", - "unicode.Canadian_Aboriginal": "unicode", - "unicode.Carian": "unicode", - "unicode.CaseRange": "unicode", - "unicode.CaseRanges": "unicode", - "unicode.Categories": "unicode", - "unicode.Caucasian_Albanian": "unicode", - "unicode.Cc": "unicode", - "unicode.Cf": "unicode", - "unicode.Chakma": "unicode", - "unicode.Cham": "unicode", - "unicode.Cherokee": "unicode", - "unicode.Co": "unicode", - "unicode.Common": "unicode", - "unicode.Coptic": "unicode", - "unicode.Cs": "unicode", - "unicode.Cuneiform": "unicode", - "unicode.Cypriot": "unicode", - "unicode.Cyrillic": "unicode", - "unicode.Dash": "unicode", - "unicode.Deprecated": "unicode", - "unicode.Deseret": "unicode", - "unicode.Devanagari": "unicode", - "unicode.Diacritic": "unicode", - "unicode.Digit": "unicode", - "unicode.Duployan": "unicode", - "unicode.Egyptian_Hieroglyphs": "unicode", - "unicode.Elbasan": "unicode", - "unicode.Ethiopic": "unicode", - "unicode.Extender": "unicode", - "unicode.FoldCategory": "unicode", - "unicode.FoldScript": "unicode", - "unicode.Georgian": "unicode", - "unicode.Glagolitic": "unicode", - "unicode.Gothic": "unicode", - "unicode.Grantha": "unicode", - "unicode.GraphicRanges": "unicode", - "unicode.Greek": "unicode", - "unicode.Gujarati": "unicode", - "unicode.Gurmukhi": "unicode", - "unicode.Han": "unicode", - "unicode.Hangul": "unicode", - "unicode.Hanunoo": "unicode", - "unicode.Hatran": "unicode", - "unicode.Hebrew": "unicode", - "unicode.Hex_Digit": "unicode", - "unicode.Hiragana": "unicode", - "unicode.Hyphen": "unicode", - "unicode.IDS_Binary_Operator": "unicode", - "unicode.IDS_Trinary_Operator": "unicode", - "unicode.Ideographic": "unicode", - "unicode.Imperial_Aramaic": "unicode", - "unicode.In": "unicode", - "unicode.Inherited": "unicode", - "unicode.Inscriptional_Pahlavi": "unicode", - "unicode.Inscriptional_Parthian": "unicode", - "unicode.Is": "unicode", - "unicode.IsControl": "unicode", - "unicode.IsDigit": "unicode", - "unicode.IsGraphic": "unicode", - "unicode.IsLetter": "unicode", - "unicode.IsLower": "unicode", - "unicode.IsMark": "unicode", - "unicode.IsNumber": "unicode", - "unicode.IsOneOf": "unicode", - "unicode.IsPrint": "unicode", - "unicode.IsPunct": "unicode", - "unicode.IsSpace": "unicode", - "unicode.IsSymbol": "unicode", - "unicode.IsTitle": "unicode", - "unicode.IsUpper": "unicode", - "unicode.Javanese": "unicode", - "unicode.Join_Control": "unicode", - "unicode.Kaithi": "unicode", - "unicode.Kannada": "unicode", - "unicode.Katakana": "unicode", - "unicode.Kayah_Li": "unicode", - "unicode.Kharoshthi": "unicode", - "unicode.Khmer": "unicode", - "unicode.Khojki": "unicode", - "unicode.Khudawadi": "unicode", - "unicode.L": "unicode", - "unicode.Lao": "unicode", - "unicode.Latin": "unicode", - "unicode.Lepcha": "unicode", - "unicode.Letter": "unicode", - "unicode.Limbu": "unicode", - "unicode.Linear_A": "unicode", - "unicode.Linear_B": "unicode", - "unicode.Lisu": "unicode", - "unicode.Ll": "unicode", - "unicode.Lm": "unicode", - "unicode.Lo": "unicode", - "unicode.Logical_Order_Exception": "unicode", - "unicode.Lower": "unicode", - "unicode.LowerCase": "unicode", - "unicode.Lt": "unicode", - "unicode.Lu": "unicode", - "unicode.Lycian": "unicode", - "unicode.Lydian": "unicode", - "unicode.M": "unicode", - "unicode.Mahajani": "unicode", - "unicode.Malayalam": "unicode", - "unicode.Mandaic": "unicode", - "unicode.Manichaean": "unicode", - "unicode.Marchen": "unicode", - "unicode.Mark": "unicode", - "unicode.MaxASCII": "unicode", - "unicode.MaxCase": "unicode", - "unicode.MaxLatin1": "unicode", - "unicode.MaxRune": "unicode", - "unicode.Mc": "unicode", - "unicode.Me": "unicode", - "unicode.Meetei_Mayek": "unicode", - "unicode.Mende_Kikakui": "unicode", - "unicode.Meroitic_Cursive": "unicode", - "unicode.Meroitic_Hieroglyphs": "unicode", - "unicode.Miao": "unicode", - "unicode.Mn": "unicode", - "unicode.Modi": "unicode", - "unicode.Mongolian": "unicode", - "unicode.Mro": "unicode", - "unicode.Multani": "unicode", - "unicode.Myanmar": "unicode", - "unicode.N": "unicode", - "unicode.Nabataean": "unicode", - "unicode.Nd": "unicode", - "unicode.New_Tai_Lue": "unicode", - "unicode.Newa": "unicode", - "unicode.Nko": "unicode", - "unicode.Nl": "unicode", - "unicode.No": "unicode", - "unicode.Noncharacter_Code_Point": "unicode", - "unicode.Number": "unicode", - "unicode.Ogham": "unicode", - "unicode.Ol_Chiki": "unicode", - "unicode.Old_Hungarian": "unicode", - "unicode.Old_Italic": "unicode", - "unicode.Old_North_Arabian": "unicode", - "unicode.Old_Permic": "unicode", - "unicode.Old_Persian": "unicode", - "unicode.Old_South_Arabian": "unicode", - "unicode.Old_Turkic": "unicode", - "unicode.Oriya": "unicode", - "unicode.Osage": "unicode", - "unicode.Osmanya": "unicode", - "unicode.Other": "unicode", - "unicode.Other_Alphabetic": "unicode", - "unicode.Other_Default_Ignorable_Code_Point": "unicode", - "unicode.Other_Grapheme_Extend": "unicode", - "unicode.Other_ID_Continue": "unicode", - "unicode.Other_ID_Start": "unicode", - "unicode.Other_Lowercase": "unicode", - "unicode.Other_Math": "unicode", - "unicode.Other_Uppercase": "unicode", - "unicode.P": "unicode", - "unicode.Pahawh_Hmong": "unicode", - "unicode.Palmyrene": "unicode", - "unicode.Pattern_Syntax": "unicode", - "unicode.Pattern_White_Space": "unicode", - "unicode.Pau_Cin_Hau": "unicode", - "unicode.Pc": "unicode", - "unicode.Pd": "unicode", - "unicode.Pe": "unicode", - "unicode.Pf": "unicode", - "unicode.Phags_Pa": "unicode", - "unicode.Phoenician": "unicode", - "unicode.Pi": "unicode", - "unicode.Po": "unicode", - "unicode.Prepended_Concatenation_Mark": "unicode", - "unicode.PrintRanges": "unicode", - "unicode.Properties": "unicode", - "unicode.Ps": "unicode", - "unicode.Psalter_Pahlavi": "unicode", - "unicode.Punct": "unicode", - "unicode.Quotation_Mark": "unicode", - "unicode.Radical": "unicode", - "unicode.Range16": "unicode", - "unicode.Range32": "unicode", - "unicode.RangeTable": "unicode", - "unicode.Rejang": "unicode", - "unicode.ReplacementChar": "unicode", - "unicode.Runic": "unicode", - "unicode.S": "unicode", - "unicode.STerm": "unicode", - "unicode.Samaritan": "unicode", - "unicode.Saurashtra": "unicode", - "unicode.Sc": "unicode", - "unicode.Scripts": "unicode", - "unicode.Sentence_Terminal": "unicode", - "unicode.Sharada": "unicode", - "unicode.Shavian": "unicode", - "unicode.Siddham": "unicode", - "unicode.SignWriting": "unicode", - "unicode.SimpleFold": "unicode", - "unicode.Sinhala": "unicode", - "unicode.Sk": "unicode", - "unicode.Sm": "unicode", - "unicode.So": "unicode", - "unicode.Soft_Dotted": "unicode", - "unicode.Sora_Sompeng": "unicode", - "unicode.Space": "unicode", - "unicode.SpecialCase": "unicode", - "unicode.Sundanese": "unicode", - "unicode.Syloti_Nagri": "unicode", - "unicode.Symbol": "unicode", - "unicode.Syriac": "unicode", - "unicode.Tagalog": "unicode", - "unicode.Tagbanwa": "unicode", - "unicode.Tai_Le": "unicode", - "unicode.Tai_Tham": "unicode", - "unicode.Tai_Viet": "unicode", - "unicode.Takri": "unicode", - "unicode.Tamil": "unicode", - "unicode.Tangut": "unicode", - "unicode.Telugu": "unicode", - "unicode.Terminal_Punctuation": "unicode", - "unicode.Thaana": "unicode", - "unicode.Thai": "unicode", - "unicode.Tibetan": "unicode", - "unicode.Tifinagh": "unicode", - "unicode.Tirhuta": "unicode", - "unicode.Title": "unicode", - "unicode.TitleCase": "unicode", - "unicode.To": "unicode", - "unicode.ToLower": "unicode", - "unicode.ToTitle": "unicode", - "unicode.ToUpper": "unicode", - "unicode.TurkishCase": "unicode", - "unicode.Ugaritic": "unicode", - "unicode.Unified_Ideograph": "unicode", - "unicode.Upper": "unicode", - "unicode.UpperCase": "unicode", - "unicode.UpperLower": "unicode", - "unicode.Vai": "unicode", - "unicode.Variation_Selector": "unicode", - "unicode.Version": "unicode", - "unicode.Warang_Citi": "unicode", - "unicode.White_Space": "unicode", - "unicode.Yi": "unicode", - "unicode.Z": "unicode", - "unicode.Zl": "unicode", - "unicode.Zp": "unicode", - "unicode.Zs": "unicode", - "url.Error": "net/url", - "url.EscapeError": "net/url", - "url.InvalidHostError": "net/url", - "url.Parse": "net/url", - "url.ParseQuery": "net/url", - "url.ParseRequestURI": "net/url", - "url.QueryEscape": "net/url", - "url.QueryUnescape": "net/url", - "url.URL": "net/url", - "url.User": "net/url", - "url.UserPassword": "net/url", - "url.Userinfo": "net/url", - "url.Values": "net/url", - "user.Current": "os/user", - "user.Group": "os/user", - "user.Lookup": "os/user", - "user.LookupGroup": "os/user", - "user.LookupGroupId": "os/user", - "user.LookupId": "os/user", - "user.UnknownGroupError": "os/user", - "user.UnknownGroupIdError": "os/user", - "user.UnknownUserError": "os/user", - "user.UnknownUserIdError": "os/user", - "user.User": "os/user", - "utf16.Decode": "unicode/utf16", - "utf16.DecodeRune": "unicode/utf16", - "utf16.Encode": "unicode/utf16", - "utf16.EncodeRune": "unicode/utf16", - "utf16.IsSurrogate": "unicode/utf16", - "utf8.DecodeLastRune": "unicode/utf8", - "utf8.DecodeLastRuneInString": "unicode/utf8", - "utf8.DecodeRune": "unicode/utf8", - "utf8.DecodeRuneInString": "unicode/utf8", - "utf8.EncodeRune": "unicode/utf8", - "utf8.FullRune": "unicode/utf8", - "utf8.FullRuneInString": "unicode/utf8", - "utf8.MaxRune": "unicode/utf8", - "utf8.RuneCount": "unicode/utf8", - "utf8.RuneCountInString": "unicode/utf8", - "utf8.RuneError": "unicode/utf8", - "utf8.RuneLen": "unicode/utf8", - "utf8.RuneSelf": "unicode/utf8", - "utf8.RuneStart": "unicode/utf8", - "utf8.UTFMax": "unicode/utf8", - "utf8.Valid": "unicode/utf8", - "utf8.ValidRune": "unicode/utf8", - "utf8.ValidString": "unicode/utf8", - "x509.CANotAuthorizedForThisName": "crypto/x509", - "x509.CertPool": "crypto/x509", - "x509.Certificate": "crypto/x509", - "x509.CertificateInvalidError": "crypto/x509", - "x509.CertificateRequest": "crypto/x509", - "x509.ConstraintViolationError": "crypto/x509", - "x509.CreateCertificate": "crypto/x509", - "x509.CreateCertificateRequest": "crypto/x509", - "x509.DSA": "crypto/x509", - "x509.DSAWithSHA1": "crypto/x509", - "x509.DSAWithSHA256": "crypto/x509", - "x509.DecryptPEMBlock": "crypto/x509", - "x509.ECDSA": "crypto/x509", - "x509.ECDSAWithSHA1": "crypto/x509", - "x509.ECDSAWithSHA256": "crypto/x509", - "x509.ECDSAWithSHA384": "crypto/x509", - "x509.ECDSAWithSHA512": "crypto/x509", - "x509.EncryptPEMBlock": "crypto/x509", - "x509.ErrUnsupportedAlgorithm": "crypto/x509", - "x509.Expired": "crypto/x509", - "x509.ExtKeyUsage": "crypto/x509", - "x509.ExtKeyUsageAny": "crypto/x509", - "x509.ExtKeyUsageClientAuth": "crypto/x509", - "x509.ExtKeyUsageCodeSigning": "crypto/x509", - "x509.ExtKeyUsageEmailProtection": "crypto/x509", - "x509.ExtKeyUsageIPSECEndSystem": "crypto/x509", - "x509.ExtKeyUsageIPSECTunnel": "crypto/x509", - "x509.ExtKeyUsageIPSECUser": "crypto/x509", - "x509.ExtKeyUsageMicrosoftServerGatedCrypto": "crypto/x509", - "x509.ExtKeyUsageNetscapeServerGatedCrypto": "crypto/x509", - "x509.ExtKeyUsageOCSPSigning": "crypto/x509", - "x509.ExtKeyUsageServerAuth": "crypto/x509", - "x509.ExtKeyUsageTimeStamping": "crypto/x509", - "x509.HostnameError": "crypto/x509", - "x509.IncompatibleUsage": "crypto/x509", - "x509.IncorrectPasswordError": "crypto/x509", - "x509.InsecureAlgorithmError": "crypto/x509", - "x509.InvalidReason": "crypto/x509", - "x509.IsEncryptedPEMBlock": "crypto/x509", - "x509.KeyUsage": "crypto/x509", - "x509.KeyUsageCRLSign": "crypto/x509", - "x509.KeyUsageCertSign": "crypto/x509", - "x509.KeyUsageContentCommitment": "crypto/x509", - "x509.KeyUsageDataEncipherment": "crypto/x509", - "x509.KeyUsageDecipherOnly": "crypto/x509", - "x509.KeyUsageDigitalSignature": "crypto/x509", - "x509.KeyUsageEncipherOnly": "crypto/x509", - "x509.KeyUsageKeyAgreement": "crypto/x509", - "x509.KeyUsageKeyEncipherment": "crypto/x509", - "x509.MD2WithRSA": "crypto/x509", - "x509.MD5WithRSA": "crypto/x509", - "x509.MarshalECPrivateKey": "crypto/x509", - "x509.MarshalPKCS1PrivateKey": "crypto/x509", - "x509.MarshalPKIXPublicKey": "crypto/x509", - "x509.NewCertPool": "crypto/x509", - "x509.NotAuthorizedToSign": "crypto/x509", - "x509.PEMCipher": "crypto/x509", - "x509.PEMCipher3DES": "crypto/x509", - "x509.PEMCipherAES128": "crypto/x509", - "x509.PEMCipherAES192": "crypto/x509", - "x509.PEMCipherAES256": "crypto/x509", - "x509.PEMCipherDES": "crypto/x509", - "x509.ParseCRL": "crypto/x509", - "x509.ParseCertificate": "crypto/x509", - "x509.ParseCertificateRequest": "crypto/x509", - "x509.ParseCertificates": "crypto/x509", - "x509.ParseDERCRL": "crypto/x509", - "x509.ParseECPrivateKey": "crypto/x509", - "x509.ParsePKCS1PrivateKey": "crypto/x509", - "x509.ParsePKCS8PrivateKey": "crypto/x509", - "x509.ParsePKIXPublicKey": "crypto/x509", - "x509.PublicKeyAlgorithm": "crypto/x509", - "x509.RSA": "crypto/x509", - "x509.SHA1WithRSA": "crypto/x509", - "x509.SHA256WithRSA": "crypto/x509", - "x509.SHA384WithRSA": "crypto/x509", - "x509.SHA512WithRSA": "crypto/x509", - "x509.SignatureAlgorithm": "crypto/x509", - "x509.SystemCertPool": "crypto/x509", - "x509.SystemRootsError": "crypto/x509", - "x509.TooManyIntermediates": "crypto/x509", - "x509.UnhandledCriticalExtension": "crypto/x509", - "x509.UnknownAuthorityError": "crypto/x509", - "x509.UnknownPublicKeyAlgorithm": "crypto/x509", - "x509.UnknownSignatureAlgorithm": "crypto/x509", - "x509.VerifyOptions": "crypto/x509", - "xml.Attr": "encoding/xml", - "xml.CharData": "encoding/xml", - "xml.Comment": "encoding/xml", - "xml.CopyToken": "encoding/xml", - "xml.Decoder": "encoding/xml", - "xml.Directive": "encoding/xml", - "xml.Encoder": "encoding/xml", - "xml.EndElement": "encoding/xml", - "xml.Escape": "encoding/xml", - "xml.EscapeText": "encoding/xml", - "xml.HTMLAutoClose": "encoding/xml", - "xml.HTMLEntity": "encoding/xml", - "xml.Header": "encoding/xml", - "xml.Marshal": "encoding/xml", - "xml.MarshalIndent": "encoding/xml", - "xml.Marshaler": "encoding/xml", - "xml.MarshalerAttr": "encoding/xml", - "xml.Name": "encoding/xml", - "xml.NewDecoder": "encoding/xml", - "xml.NewEncoder": "encoding/xml", - "xml.ProcInst": "encoding/xml", - "xml.StartElement": "encoding/xml", - "xml.SyntaxError": "encoding/xml", - "xml.TagPathError": "encoding/xml", - "xml.Token": "encoding/xml", - "xml.Unmarshal": "encoding/xml", - "xml.UnmarshalError": "encoding/xml", - "xml.Unmarshaler": "encoding/xml", - "xml.UnmarshalerAttr": "encoding/xml", - "xml.UnsupportedTypeError": "encoding/xml", - "zip.Compressor": "archive/zip", - "zip.Decompressor": "archive/zip", - "zip.Deflate": "archive/zip", - "zip.ErrAlgorithm": "archive/zip", - "zip.ErrChecksum": "archive/zip", - "zip.ErrFormat": "archive/zip", - "zip.File": "archive/zip", - "zip.FileHeader": "archive/zip", - "zip.FileInfoHeader": "archive/zip", - "zip.NewReader": "archive/zip", - "zip.NewWriter": "archive/zip", - "zip.OpenReader": "archive/zip", - "zip.ReadCloser": "archive/zip", - "zip.Reader": "archive/zip", - "zip.RegisterCompressor": "archive/zip", - "zip.RegisterDecompressor": "archive/zip", - "zip.Store": "archive/zip", - "zip.Writer": "archive/zip", - "zlib.BestCompression": "compress/zlib", - "zlib.BestSpeed": "compress/zlib", - "zlib.DefaultCompression": "compress/zlib", - "zlib.ErrChecksum": "compress/zlib", - "zlib.ErrDictionary": "compress/zlib", - "zlib.ErrHeader": "compress/zlib", - "zlib.NewReader": "compress/zlib", - "zlib.NewReaderDict": "compress/zlib", - "zlib.NewWriter": "compress/zlib", - "zlib.NewWriterLevel": "compress/zlib", - "zlib.NewWriterLevelDict": "compress/zlib", - "zlib.NoCompression": "compress/zlib", - "zlib.Resetter": "compress/zlib", - "zlib.Writer": "compress/zlib", - - "unsafe.Alignof": "unsafe", - "unsafe.ArbitraryType": "unsafe", - "unsafe.Offsetof": "unsafe", - "unsafe.Pointer": "unsafe", - "unsafe.Sizeof": "unsafe", -} diff --git a/vendor/vendor.json b/vendor/vendor.json index 6e5610460a..1bfe09da72 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -38,6 +38,18 @@ "revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338", "revisionTime": "2018-01-16T20:38:02Z" }, + { + "checksumSHA1": "9Niiu1GNhWUrXnGZrl8AU4EzbVE=", + "path": "github.com/allegro/bigcache", + "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca", + "revisionTime": "2018-10-22T20:06:25Z" + }, + { + "checksumSHA1": "zqToN+R6KybEskp1D4G/lAOKXU4=", + "path": "github.com/allegro/bigcache/queue", + "revision": "bff00e20c68d9f136477d62d182a7dc917bae0ca", + "revisionTime": "2018-10-22T20:06:25Z" + }, { "checksumSHA1": "USkefO0g1U9mr+8hagv3fpSkrxg=", "path": "github.com/aristanetworks/goarista/monotime", @@ -268,22 +280,23 @@ "revisionTime": "2016-07-20T14:16:34Z" }, { - "checksumSHA1": "I4njd26dG5hxFT2nawuByM4pxzY=", + "checksumSHA1": "SEnjvwVyfuU2xBaOfXfwPD5MZqk=", "path": "github.com/mattn/go-colorable", - "revision": "5411d3eea5978e6cdc258b30de592b60df6aba96", - "revisionTime": "2017-02-10T17:28:01Z" + "revision": "efa589957cd060542a26d2dd7832fd6a6c6c3ade", + "revisionTime": "2018-03-10T13:32:14Z", + "version": "efa589957cd060542a26d2dd7832fd6a6c6c3ade" }, { - "checksumSHA1": "EkT5JmFvz3zOPWappEFyYWUaeY0=", + "checksumSHA1": "GiVgQkx5acnq+JZtYiuHPlhHoso=", "path": "github.com/mattn/go-isatty", - "revision": "281032e84ae07510239465db46bf442aa44b953a", - "revisionTime": "2017-02-09T17:56:15Z" + "revision": "3fb116b820352b7f0c281308a4d6250c22d94e27", + "revisionTime": "2018-08-30T10:17:45Z" }, { "checksumSHA1": "MNkKJyk2TazKMJYbal5wFHybpyA=", "path": "github.com/mattn/go-runewidth", - "revision": "14207d285c6c197daabb5c9793d63e7af9ab2d50", - "revisionTime": "2017-02-01T02:35:40Z" + "revision": "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb", + "revisionTime": "2018-04-08T05:53:51Z" }, { "checksumSHA1": "L3leymg2RT8hFl5uL+5KP/LpBkg=", @@ -879,12 +892,6 @@ "revision": "be0fcc31ae2332374e800dfff29b721c585b35df", "revisionTime": "2016-11-04T18:56:24Z" }, - { - "checksumSHA1": "2ko3hvt1vrfwG+p7SLW+zqDEeV4=", - "path": "golang.org/x/tools/imports", - "revision": "be0fcc31ae2332374e800dfff29b721c585b35df", - "revisionTime": "2016-11-04T18:56:24Z" - }, { "checksumSHA1": "CEFTYXtWmgSh+3Ik1NmDaJcz4E0=", "path": "gopkg.in/check.v1",