From fa16d497bcf45719edf9e4e7a1823449f6562ea8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 27 Feb 2025 12:07:36 +0100 Subject: [PATCH 1/4] build: update PPA Go bootstrap version to 1.23 (#31282) This is for fixing the PPA build, which has been failing since the update to Go 1.24. In Go 1.24, the required Go version for bootstrapping was updated to 1.22. In general, they are following through with always depending on the Go version two releases ago for bootstrapping. Since we still support Ubuntu Xenial (16.04) until its EOL date of 04/2026, and Xenial only has golang 1.10 as a package, we now need to build Go a total of four times to get the most recent version. I'm adding a step for Go 1.23 here. This should last us until Go 1.25, which should be out around 04/2026, and we can hopefully drop the first bootstrapping step at that time. --- build/checksums.txt | 8 ++++++-- build/ci.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 1646cc6c45..c8b742e443 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -114,14 +114,18 @@ fc43f8c95d6bec8ba9a3557a0bb63cf9f504137ce6d24cfdc4c330d46f7779d2 golangci-lint- # This version is fine to be old and full of security holes, we just use it # to build the latest Go. Don't change it. # -# version:ppa-builder-1 1.19.6 +# version:ppa-builder-1.19 1.19.6 # https://go.dev/dl/ d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767 go1.19.6.src.tar.gz -# version:ppa-builder-2 1.21.9 +# version:ppa-builder-1.21 1.21.9 # https://go.dev/dl/ 58f0c5ced45a0012bce2ff7a9df03e128abcc8818ebabe5027bb92bafe20e421 go1.21.9.src.tar.gz +# version:ppa-builder-1.23 1.23.6 +# https://go.dev/dl/ +039c5b04e65279daceee8a6f71e70bd05cf5b801782b6f77c6e19e2ed0511222 go1.23.6.src.tar.gz + # version:protoc 27.1 # https://github.com/protocolbuffers/protobuf/releases/ # https://github.com/protocolbuffers/protobuf/releases/download/v27.1/ diff --git a/build/ci.go b/build/ci.go index fbf739e476..6203af98ba 100644 --- a/build/ci.go +++ b/build/ci.go @@ -843,7 +843,7 @@ func downloadGoBootstrapSources(cachedir string) []string { csdb := build.MustLoadChecksums("build/checksums.txt") var bundles []string - for _, booter := range []string{"ppa-builder-1", "ppa-builder-2"} { + for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} { gobootVersion, err := build.Version(csdb, booter) if err != nil { log.Fatal(err) From f005d95b55a3cc1d3c017ec19e949e842aaf0c33 Mon Sep 17 00:00:00 2001 From: Shude Li Date: Thu, 27 Feb 2025 22:52:00 +0800 Subject: [PATCH 2/4] build: simplify go mod tidy check (#31266) This changes the go mod tidy check to use the go mod tidy -diff command, removing the custom diffing for go.mod. The check for go.mod/go.sum is now performed in the check_generate action. Also included is a change where check_generate and check_baddeps will now run on the GitHub Actions lint step. --------- Co-authored-by: Felix Lange --- .github/workflows/go.yml | 2 +- appveyor.yml | 1 - build/ci.go | 28 +++++++--------------------- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4da92f2431..78410aab10 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -29,7 +29,7 @@ jobs: - name: Run linters run: | go run build/ci.go lint - go run build/ci.go check_tidy + go run build/ci.go check_generate go run build/ci.go check_baddeps build: diff --git a/appveyor.yml b/appveyor.yml index 1543211edc..ae1c74c18e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,7 +24,6 @@ for: - image: Ubuntu build_script: - go run build/ci.go lint - - go run build/ci.go check_tidy - go run build/ci.go check_generate - go run build/ci.go check_baddeps - go run build/ci.go install -dlgo diff --git a/build/ci.go b/build/ci.go index 6203af98ba..b2ff829d3f 100644 --- a/build/ci.go +++ b/build/ci.go @@ -25,8 +25,7 @@ Usage: go run build/ci.go Available commands are: lint -- runs certain pre-selected linters - check_tidy -- verifies that everything is 'go mod tidy'-ed - check_generate -- verifies that everything is 'go generate'-ed + check_generate -- verifies that 'go generate' and 'go mod tidy' do not produce changes check_baddeps -- verifies that certain dependencies are avoided install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables @@ -155,8 +154,6 @@ func main() { doTest(os.Args[2:]) case "lint": doLint(os.Args[2:]) - case "check_tidy": - doCheckTidy() case "check_generate": doCheckGenerate() case "check_baddeps": @@ -352,22 +349,6 @@ func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string { // doCheckTidy assets that the Go modules files are tidied already. func doCheckTidy() { - targets := []string{"go.mod", "go.sum"} - - hashes, err := build.HashFiles(targets) - if err != nil { - log.Fatalf("failed to hash go.mod/go.sum: %v", err) - } - build.MustRun(new(build.GoToolchain).Go("mod", "tidy")) - - tidied, err := build.HashFiles(targets) - if err != nil { - log.Fatalf("failed to rehash go.mod/go.sum: %v", err) - } - if updates := build.DiffHashes(hashes, tidied); len(updates) > 0 { - log.Fatalf("files changed on running 'go mod tidy': %v", updates) - } - fmt.Println("No untidy module files detected.") } // doCheckGenerate ensures that re-generating generated files does not cause @@ -375,6 +356,7 @@ func doCheckTidy() { func doCheckGenerate() { var ( cachedir = flag.String("cachedir", "./build/cache", "directory for caching binaries.") + tc = new(build.GoToolchain) ) // Compute the origin hashes of all the files var hashes map[string][32]byte @@ -389,7 +371,7 @@ func doCheckGenerate() { protocPath = downloadProtoc(*cachedir) protocGenGoPath = downloadProtocGenGo(*cachedir) ) - c := new(build.GoToolchain).Go("generate", "./...") + c := tc.Go("generate", "./...") pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")} c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator))) build.MustRun(c) @@ -407,6 +389,10 @@ func doCheckGenerate() { log.Fatal("One or more generated files were updated by running 'go generate ./...'") } fmt.Println("No stale files detected.") + + // Run go mod tidy check. + build.MustRun(tc.Go("mod", "tidy", "-diff")) + fmt.Println("No untidy module files detected.") } // doCheckBadDeps verifies whether certain unintended dependencies between some From 767c202e4779536f4d9d862080304f20d9514371 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Thu, 27 Feb 2025 15:53:52 +0100 Subject: [PATCH 3/4] all: drop x/exp direct dependency (#30558) This is a not-particularly-important "cleanliness" PR. It removes the last remnants of the `x/exp` package, where we used the `maps.Keys` function. The original returned the keys in a slice, but when it became 'native' the signature changed to return an iterator, so the new idiom is `slices.Collect(maps.Keys(theMap))`, unless of course the raw iterator can be used instead. In some cases, where we previously collect into slice and then sort, we can now instead do `slices.SortXX` on the iterator instead, making the code a bit more concise. This PR might be _slighly_ less optimal, because the original `x/exp` implementation allocated the slice at the correct size off the bat, which I suppose the new code won't. Putting it up for discussion. --------- Co-authored-by: Felix Lange --- cmd/devp2p/internal/ethtest/chain.go | 6 +++--- cmd/evm/blockrunner.go | 5 ++--- core/state/snapshot/difflayer.go | 8 +++----- core/txpool/blobpool/evictheap.go | 5 ++--- core/txpool/legacypool/legacypool.go | 4 ++-- core/txpool/locals/tx_tracker.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- p2p/netutil/net.go | 6 ++---- triedb/pathdb/history.go | 10 +++------- triedb/pathdb/states.go | 8 +++----- 11 files changed, 24 insertions(+), 36 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index a90c53c432..689667a56b 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + "maps" "math/big" "os" "path/filepath" @@ -40,7 +41,6 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/exp/maps" ) // Chain is a lightweight blockchain-like store which can read a hivechain @@ -162,8 +162,8 @@ func (c *Chain) RootAt(height int) common.Hash { // GetSender returns the address associated with account at the index in the // pre-funded accounts list. func (c *Chain) GetSender(idx int) (common.Address, uint64) { - accounts := maps.Keys(c.senders) - slices.SortFunc(accounts, common.Address.Cmp) + accounts := slices.SortedFunc(maps.Keys(c.senders), common.Address.Cmp) + addr := accounts[idx] return addr, c.senders[addr].Nonce } diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index 9a04807b33..31d1ba5ba1 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" "regexp" "slices" @@ -28,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" - "golang.org/x/exp/maps" ) var blockTestCommand = &cli.Command{ @@ -80,8 +80,7 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { tracer := tracerFromFlags(ctx) // Pull out keys to sort and ensure tests are run in order. - keys := maps.Keys(tests) - slices.Sort(keys) + keys := slices.Sorted(maps.Keys(tests)) // Run all the tests. var results []testResult diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index dce4f79a11..28957051d4 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -19,6 +19,7 @@ package snapshot import ( "encoding/binary" "fmt" + "maps" "math" "math/rand" "slices" @@ -30,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" bloomfilter "github.com/holiman/bloomfilter/v2" - "golang.org/x/exp/maps" ) var ( @@ -431,8 +431,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.lock.Lock() defer dl.lock.Unlock() - dl.accountList = maps.Keys(dl.accountData) - slices.SortFunc(dl.accountList, common.Hash.Cmp) + dl.accountList = slices.SortedFunc(maps.Keys(dl.accountData), common.Hash.Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList } @@ -464,8 +463,7 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { dl.lock.Lock() defer dl.lock.Unlock() - storageList := maps.Keys(dl.storageData[accountHash]) - slices.SortFunc(storageList, common.Hash.Cmp) + storageList := slices.SortedFunc(maps.Keys(dl.storageData[accountHash]), common.Hash.Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) return storageList diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index 5e285e6c53..722a71bc9b 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -18,12 +18,12 @@ package blobpool import ( "container/heap" + "maps" "math" "slices" "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" - "golang.org/x/exp/maps" ) // evictHeap is a helper data structure to keep track of the cheapest bottleneck @@ -54,8 +54,7 @@ func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index map[common.A // Populate the heap in account sort order. Not really needed in practice, // but it makes the heap initialization deterministic and less annoying to // test in unit tests. - heap.addrs = maps.Keys(index) - slices.SortFunc(heap.addrs, common.Address.Cmp) + heap.addrs = slices.SortedFunc(maps.Keys(index), common.Address.Cmp) for i, addr := range heap.addrs { heap.index[addr] = i } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 0fda56986e..fb10060496 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -19,6 +19,7 @@ package legacypool import ( "errors" + "maps" "math" "math/big" "slices" @@ -40,7 +41,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" - "golang.org/x/exp/maps" ) const ( @@ -1674,7 +1674,7 @@ func (as *accountSet) addTx(tx *types.Transaction) { // reuse. The returned slice should not be changed! func (as *accountSet) flatten() []common.Address { if as.cache == nil { - as.cache = maps.Keys(as.accounts) + as.cache = slices.Collect(maps.Keys(as.accounts)) } return as.cache } diff --git a/core/txpool/locals/tx_tracker.go b/core/txpool/locals/tx_tracker.go index a24fcb1f4e..0113646669 100644 --- a/core/txpool/locals/tx_tracker.go +++ b/core/txpool/locals/tx_tracker.go @@ -18,6 +18,7 @@ package locals import ( + "slices" "sync" "time" @@ -28,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" - "golang.org/x/exp/slices" ) var ( diff --git a/go.mod b/go.mod index e58d8fd828..ba26f2150e 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,6 @@ require ( go.uber.org/automaxprocs v1.5.2 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.32.0 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.10.0 golang.org/x/sys v0.29.0 golang.org/x/text v0.21.0 @@ -142,6 +141,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.34.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 0dbd79cfaa..1e4efeb697 100644 --- a/go.sum +++ b/go.sum @@ -553,8 +553,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go index 7d8da88670..696c331859 100644 --- a/p2p/netutil/net.go +++ b/p2p/netutil/net.go @@ -21,12 +21,11 @@ import ( "bytes" "errors" "fmt" + "maps" "net" "net/netip" "slices" "strings" - - "golang.org/x/exp/maps" ) var special4, special6 Netlist @@ -324,8 +323,7 @@ func (s *DistinctNetSet) key(ip netip.Addr) netip.Prefix { // String implements fmt.Stringer func (s DistinctNetSet) String() string { - keys := maps.Keys(s.members) - slices.SortFunc(keys, func(a, b netip.Prefix) int { + keys := slices.SortedFunc(maps.Keys(s.members), func(a, b netip.Prefix) int { return strings.Compare(a.String(), b.String()) }) diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 6ba9aaf1bb..c063e45371 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "maps" "slices" "time" @@ -29,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" - "golang.org/x/exp/maps" ) // State history records the state changes involved in executing a block. The @@ -250,15 +250,11 @@ type history struct { // newHistory constructs the state history object with provided state change set. func newHistory(root common.Hash, parent common.Hash, block uint64, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, rawStorageKey bool) *history { var ( - accountList = maps.Keys(accounts) + accountList = slices.SortedFunc(maps.Keys(accounts), common.Address.Cmp) storageList = make(map[common.Address][]common.Hash) ) - slices.SortFunc(accountList, common.Address.Cmp) - for addr, slots := range storages { - slist := maps.Keys(slots) - slices.SortFunc(slist, common.Hash.Cmp) - storageList[addr] = slist + storageList[addr] = slices.SortedFunc(maps.Keys(slots), common.Hash.Cmp) } version := historyVersion if !rawStorageKey { diff --git a/triedb/pathdb/states.go b/triedb/pathdb/states.go index 873a7ba618..0a83b2f2cc 100644 --- a/triedb/pathdb/states.go +++ b/triedb/pathdb/states.go @@ -19,6 +19,7 @@ package pathdb import ( "fmt" "io" + "maps" "slices" "sync" @@ -27,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/exp/maps" ) // counter helps in tracking items and their corresponding sizes. @@ -174,8 +174,7 @@ func (s *stateSet) accountList() []common.Hash { s.listLock.Lock() defer s.listLock.Unlock() - list = maps.Keys(s.accountData) - slices.SortFunc(list, common.Hash.Cmp) + list = slices.SortedFunc(maps.Keys(s.accountData), common.Hash.Cmp) s.accountListSorted = list return list } @@ -205,8 +204,7 @@ func (s *stateSet) storageList(accountHash common.Hash) []common.Hash { s.listLock.Lock() defer s.listLock.Unlock() - list := maps.Keys(s.storageData[accountHash]) - slices.SortFunc(list, common.Hash.Cmp) + list := slices.SortedFunc(maps.Keys(s.storageData[accountHash]), common.Hash.Cmp) s.storageListSorted[accountHash] = list return list } From 6c286beb39f61c9457c19deb0aa90bb172d80505 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 27 Feb 2025 17:52:22 +0100 Subject: [PATCH 4/4] build/deb: add step for new Go bootstrap to debian rules (#31283) Next attempt at fixing the build on launchpad.net --- build/deb/ethereum/deb.rules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 3287e15ff0..5905ad50a7 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -21,7 +21,8 @@ override_dh_auto_build: # requirements opposed to older versions of Go. (mv .goboot-1 ../ && cd ../.goboot-1/src && ./make.bash) (mv .goboot-2 ../ && cd ../.goboot-2/src && GOROOT_BOOTSTRAP=`pwd`/../../.goboot-1 ./make.bash) - (mv .go ../ && cd ../.go/src && GOROOT_BOOTSTRAP=`pwd`/../../.goboot-2 ./make.bash) + (mv .goboot-3 ../ && cd ../.goboot-3/src && GOROOT_BOOTSTRAP=`pwd`/../../.goboot-2 ./make.bash) + (mv .go ../ && cd ../.go/src && GOROOT_BOOTSTRAP=`pwd`/../../.goboot-3 ./make.bash) # We can't download external go modules within Launchpad, so we're shipping the # entire dependency source cache with go-ethereum.