cmd/utils: max out the OS file allowance, don't cap to 2K

Backport: ethereum/go-ethereum#18211

cmd/utils/fdlimit_test.go

@@ -1,35 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package utils
-
-import "testing"
-
-// TestFileDescriptorLimits simply tests whether the file descriptor allowance
-// per this process can be retrieved.
-func TestFileDescriptorLimits(t *testing.T) {
-	target := 4096
-
-	if limit, err := getFdLimit(); err != nil || limit <= 0 {
-		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
-	}
-	if err := raiseFdLimit(uint64(target)); err != nil {
-		t.Fatalf("failed to raise file allowance")
-	}
-	if limit, err := getFdLimit(); err != nil || limit < target {
-		t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err)
-	}
-}

cmd/utils/fdlimit_unix.go

@@ -1,50 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// +build linux darwin netbsd openbsd solaris
-
-package utils
-
-import "syscall"
-
-// raiseFdLimit tries to maximize the file descriptor allowance of this process
-// to the maximum hard-limit allowed by the OS.
-func raiseFdLimit(max uint64) error {
-	// Get the current limit
-	var limit syscall.Rlimit
-	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
-	}
-	// Try to update the limit to the max allowance
-	limit.Cur = limit.Max
-	if limit.Cur > max {
-		limit.Cur = max
-	}
-	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
-	}
-	return nil
-}
-
-// getFdLimit retrieves the number of file descriptors allowed to be opened by this
-// process.
-func getFdLimit() (int, error) {
-	var limit syscall.Rlimit
-	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return 0, err
-	}
-	return int(limit.Cur), nil
-}

cmd/utils/fdlimit_windows.go

@@ -1,41 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package utils
-
-import "errors"
-
-// raiseFdLimit tries to maximize the file descriptor allowance of this process
-// to the maximum hard-limit allowed by the OS.
-func raiseFdLimit(max uint64) error {
-	// This method is NOP by design:
-	//  * Linux/Darwin counterparts need to manually increase per process limits
-	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
-	//    changeable from within a running process
-	// This way we can always "request" raising the limits, which will either have
-	// or not have effect based on the platform we're running on.
-	if max > 16384 {
-		return errors.New("file descriptor limit (16384) reached")
-	}
-	return nil
-}
-
-// getFdLimit retrieves the number of file descriptors allowed to be opened by this
-// process.
-func getFdLimit() (int, error) {
-	// Please see raiseFdLimit for the reason why we use hard coded 16K as the limit
-	return 16384, nil
-}

cmd/utils/flags.go

@@ -28,10 +28,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/ubiq/ubqhash"
 	"github.com/ubiq/go-ubiq/accounts"
 	"github.com/ubiq/go-ubiq/accounts/keystore"
 	"github.com/ubiq/go-ubiq/common"
+	"github.com/ubiq/go-ubiq/common/fdlimit"
 	"github.com/ubiq/go-ubiq/core"
 	"github.com/ubiq/go-ubiq/core/state"
 	"github.com/ubiq/go-ubiq/core/vm"
@@ -53,6 +53,7 @@ import (
 	"github.com/ubiq/go-ubiq/pow"
 	"github.com/ubiq/go-ubiq/rpc"
 	whisper "github.com/ubiq/go-ubiq/whisper/whisperv2"
+	"github.com/ubiq/ubqhash"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -565,18 +566,15 @@ func MakeWSRpcHost(ctx *cli.Context) string {
 	return ctx.GlobalString(WSListenAddrFlag.Name)
 }
 
-// MakeDatabaseHandles raises out the number of allowed file handles per process
+// makeDatabaseHandles raises out the number of allowed file handles per process
 // for Gubiq and returns half of the allowance to assign to the database.
-func MakeDatabaseHandles() int {
+func makeDatabaseHandles() int {
-	if err := raiseFdLimit(2048); err != nil {
+	limit, err := fdlimit.Maximum()
-		Fatalf("Failed to raise file descriptor allowance: %v", err)
-	}
-	limit, err := getFdLimit()
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if limit > 2048 { // cap database file descriptors even if more is available
+	if err := fdlimit.Raise(uint64(limit)); err != nil {
-		limit = 2048
+		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
 	return limit / 2 // Leave half for networking and other stuff
 }
@@ -732,7 +730,7 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 		LightPeers:              ctx.GlobalInt(LightPeersFlag.Name),
 		MaxPeers:                ctx.GlobalInt(MaxPeersFlag.Name),
 		DatabaseCache:           ctx.GlobalInt(CacheFlag.Name),
-		DatabaseHandles:         MakeDatabaseHandles(),
+		DatabaseHandles:         makeDatabaseHandles(),
 		NetworkId:               ctx.GlobalInt(NetworkIdFlag.Name),
 		MinerThreads:            ctx.GlobalInt(MinerThreadsFlag.Name),
 		ExtraData:               MakeMinerExtra(extra, ctx),
@@ -885,7 +883,7 @@ func ChainDbName(ctx *cli.Context) string {
 func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	var (
 		cache   = ctx.GlobalInt(CacheFlag.Name)
-		handles = MakeDatabaseHandles()
+		handles = makeDatabaseHandles()
 		name    = ChainDbName(ctx)
 	)
 

common/fdlimit/fdlimit_freebsd.go

@@ -1,22 +1,22 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
+// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
+// You should have received a copy of the GNU Lesser General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
 // +build freebsd
 
-package utils
+package fdlimit
 
 import "syscall"
 
@@ -24,9 +24,9 @@ import "syscall"
 // but Rlimit fields have type int64 on FreeBSD so it needs
 // an extra conversion.
 
-// raiseFdLimit tries to maximize the file descriptor allowance of this process
+// Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func raiseFdLimit(max uint64) error {
+func Raise(max uint64) error {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
@@ -43,12 +43,22 @@ func raiseFdLimit(max uint64) error {
 	return nil
 }
 
-// getFdLimit retrieves the number of file descriptors allowed to be opened by this
+// Current retrieves the number of file descriptors allowed to be opened by this
 // process.
-func getFdLimit() (int, error) {
+func Current() (int, error) {
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return 0, err
 	}
 	return int(limit.Cur), nil
 }
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Max), nil
+}

common/fdlimit/fdlimit_test.go

@@ -0,0 +1,45 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestFileDescriptorLimits simply tests whether the file descriptor allowance
+// per this process can be retrieved.
+func TestFileDescriptorLimits(t *testing.T) {
+	target := 4096
+	hardlimit, err := Maximum()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hardlimit < target {
+		t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target))
+	}
+
+	if limit, err := Current(); err != nil || limit <= 0 {
+		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
+	}
+	if err := Raise(uint64(target)); err != nil {
+		t.Fatalf("failed to raise file allowance")
+	}
+	if limit, err := Current(); err != nil || limit < target {
+		t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err)
+	}
+}

common/fdlimit/fdlimit_unix.go

@@ -0,0 +1,60 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build linux darwin netbsd openbsd solaris
+
+package fdlimit
+
+import "syscall"
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func Raise(max uint64) error {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Max), nil
+}

common/fdlimit/fdlimit_windows.go

@@ -0,0 +1,47 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import "errors"
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func Raise(max uint64) error {
+	// This method is NOP by design:
+	//  * Linux/Darwin counterparts need to manually increase per process limits
+	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
+	//    changeable from within a running process
+	// This way we can always "request" raising the limits, which will either have
+	// or not have effect based on the platform we're running on.
+	if max > 16384 {
+		return errors.New("file descriptor limit (16384) reached")
+	}
+	return nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	// Please see Raise for the reason why we use hard coded 16K as the limit
+	return 16384, nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	return Current()
+}