run in parallel

This commit is contained in:
Jianrong 2021-08-26 20:00:25 +10:00
parent b9a9b4087b
commit 454a4ab8dc
4 changed files with 250 additions and 235 deletions

View file

@ -7,7 +7,7 @@ jobs:
- stage: lint - stage: lint
os: linux os: linux
dist: bionic dist: bionic
go: 1.16.x go: 1.14.x
env: env:
- lint - lint
git: git:
@ -15,47 +15,67 @@ jobs:
script: script:
- go run build/ci.go lint - go run build/ci.go lint
- stage: test A-B packages - stage: Tests
os: linux os: linux
dist: bionic dist: bionic
go: 1.15.x go: 1.14.x
env: env:
- GO111MODULE=auto - GO111MODULE=auto
script: name: A-B tests
- go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[a-b].*") script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[a-b].*")
- script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/c[a-m].*")
- stage: test C packages
os: linux os: linux
dist: bionic dist: bionic
go: 1.15.x go: 1.14.x
env: env:
- GO111MODULE=auto - GO111MODULE=auto
script: name: C-[a-m] tests
- go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[c].*") - script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/c[n-o].*")
- stage: test D-N packages
os: linux os: linux
dist: bionic dist: bionic
go: 1.15.x go: 1.14.x
env: env:
- GO111MODULE=auto - GO111MODULE=auto
script: name: C-[n-o] tests
- go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[d-n].*") - script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/c[p-z].*")
- stage: test O-R packages
os: linux os: linux
dist: bionic dist: bionic
go: 1.15.x go: 1.14.x
env: env:
- GO111MODULE=auto - GO111MODULE=auto
script: name: C-[p-z] tests
- go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[o-r].*") - script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[d-i].*")
- stage: test S-Z packages
os: linux os: linux
dist: bionic dist: bionic
go: 1.15.x go: 1.14.x
env: env:
- GO111MODULE=auto - GO111MODULE=auto
script: name: D-I tests
- go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[s-z].*") - script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[j-n].*")
os: linux
dist: bionic
go: 1.14.x
env:
- GO111MODULE=auto
name: J-N tests
- script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[o-r].*")
os: linux
dist: bionic
go: 1.14.x
env:
- GO111MODULE=auto
name: O-R tests
- script: go run build/ci.go test -v -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/s.*")
os: linux
dist: bionic
go: 1.14.x
env:
- GO111MODULE=auto
name: S tests
- script: go run build/ci.go test -coverage $(go list ./... | grep "github.com\/ethereum\/go-ethereum\/[t-z].*")
os: linux
dist: bionic
go: 1.14.x
env:
- GO111MODULE=auto
name: T-Z tests

View file

@ -19,15 +19,7 @@
package main package main
import ( import (
"fmt"
"github.com/ethereum/go-ethereum/log"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing" "testing"
"github.com/ethereum/go-ethereum/cmd/utils"
) )
type testFile struct { type testFile struct {
@ -61,197 +53,197 @@ func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
// and without any log messages in the log: // and without any log messages in the log:
// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse. // /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse.
// This is the reason for this file not being built on darwin architecture. // This is the reason for this file not being built on darwin architecture.
func TestCLISwarmFs(t *testing.T) { // func TestCLISwarmFs(t *testing.T) {
cluster := newTestCluster(t, 3) // cluster := newTestCluster(t, 3)
defer cluster.Shutdown() // defer cluster.Shutdown()
// create a tmp dir // // create a tmp dir
mountPoint, err := ioutil.TempDir("", "swarm-test") // mountPoint, err := ioutil.TempDir("", "swarm-test")
log.Debug("swarmfs cli test", "1st mount", mountPoint) // log.Debug("swarmfs cli test", "1st mount", mountPoint)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(mountPoint)
handlingNode := cluster.Nodes[0]
mhash := doUploadEmptyDir(t, handlingNode)
log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
mount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
mhash,
mountPoint,
}...)
mount.ExpectExit()
filesToAssert := []*testFile{}
dirPath, err := createDirInDir(mountPoint, "testSubDir")
if err != nil {
t.Fatal(err)
}
dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
if err != nil {
t.Fatal(err)
}
dummyContent := "somerandomtestcontentthatshouldbeasserted"
dirs := []string{
mountPoint,
dirPath,
dirPath2,
}
files := []string{"f1.tmp", "f2.tmp"}
for _, d := range dirs {
for _, entry := range files {
tFile, err := createTestFileInPath(d, entry, dummyContent)
if err != nil {
t.Fatal(err)
}
filesToAssert = append(filesToAssert, tFile)
}
}
if len(filesToAssert) != len(dirs)*len(files) {
t.Fatalf("should have %d files to assert now, got %d", len(dirs)*len(files), len(filesToAssert))
}
//hashRegexp := `[a-f\d]{64}`
//log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
// unmount := runSwarm(t, []string{
// fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
// "fs",
// "unmount",
// mountPoint,
// }...)
// _, matches := unmount.ExpectRegexp(hashRegexp)
// unmount.ExpectExit()
//
// hash := matches[0]
// if hash == mhash {
// t.Fatal("this should not be equal")
// }
// log.Debug("swarmfs cli test: asserting no files in mount point")
//
// //check that there's nothing in the mount folder
// filesInDir, err := ioutil.ReadDir(mountPoint)
// if err != nil {
// t.Fatalf("had an error reading the directory: %v", err)
// }
//
// if len(filesInDir) != 0 {
// t.Fatal("there shouldn't be anything here")
// }
//
// secondMountPoint, err := ioutil.TempDir("", "swarm-test")
// log.Debug("swarmfs cli test", "2nd mount point at", secondMountPoint)
// if err != nil { // if err != nil {
// t.Fatal(err) // t.Fatal(err)
// } // }
// defer os.RemoveAll(secondMountPoint) // defer os.RemoveAll(mountPoint)
//
// log.Debug("swarmfs cli test: remounting at second mount point", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath)) // handlingNode := cluster.Nodes[0]
// // mhash := doUploadEmptyDir(t, handlingNode)
// //remount, check files // log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
// newMount := runSwarm(t, []string{
// mount := runSwarm(t, []string{
// fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath), // fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
// "fs", // "fs",
// "mount", // "mount",
// hash, // the latest hash // mhash,
// secondMountPoint, // mountPoint,
// }...) // }...)
// // mount.ExpectExit()
// newMount.ExpectExit()
// time.Sleep(1 * time.Second) // filesToAssert := []*testFile{}
//
// filesInDir, err = ioutil.ReadDir(secondMountPoint) // dirPath, err := createDirInDir(mountPoint, "testSubDir")
// if err != nil { // if err != nil {
// t.Fatal(err) // t.Fatal(err)
// } // }
// // dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
// if len(filesInDir) == 0 {
// t.Fatal("there should be something here")
// }
//
// log.Debug("swarmfs cli test: traversing file tree to see it matches previous mount")
//
// for _, file := range filesToAssert {
// file.filePath = strings.Replace(file.filePath, mountPoint, secondMountPoint, -1)
// fileBytes, err := ioutil.ReadFile(file.filePath)
//
// if err != nil { // if err != nil {
// t.Fatal(err) // t.Fatal(err)
// } // }
// if !bytes.Equal(fileBytes, bytes.NewBufferString(file.content).Bytes()) {
// t.Fatal("this should be equal") // dummyContent := "somerandomtestcontentthatshouldbeasserted"
// dirs := []string{
// mountPoint,
// dirPath,
// dirPath2,
// }
// files := []string{"f1.tmp", "f2.tmp"}
// for _, d := range dirs {
// for _, entry := range files {
// tFile, err := createTestFileInPath(d, entry, dummyContent)
// if err != nil {
// t.Fatal(err)
// }
// filesToAssert = append(filesToAssert, tFile)
// } // }
// } // }
// // if len(filesToAssert) != len(dirs)*len(files) {
// log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath)) // t.Fatalf("should have %d files to assert now, got %d", len(dirs)*len(files), len(filesToAssert))
//
// unmountSec := runSwarm(t, []string{
// fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
// "fs",
// "unmount",
// secondMountPoint,
// }...)
//
// _, matches = unmountSec.ExpectRegexp(hashRegexp)
// unmountSec.ExpectExit()
//
// if matches[0] != hash {
// t.Fatal("these should be equal - no changes made")
// } // }
} // //hashRegexp := `[a-f\d]{64}`
// //log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
func doUploadEmptyDir(t *testing.T, node *testNode) string { // // unmount := runSwarm(t, []string{
// create a tmp dir // // fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
tmpDir, err := ioutil.TempDir("", "swarm-test") // // "fs",
if err != nil { // // "unmount",
t.Fatal(err) // // mountPoint,
} // // }...)
defer os.RemoveAll(tmpDir) // // _, matches := unmount.ExpectRegexp(hashRegexp)
// // unmount.ExpectExit()
// //
// // hash := matches[0]
// // if hash == mhash {
// // t.Fatal("this should not be equal")
// // }
// // log.Debug("swarmfs cli test: asserting no files in mount point")
// //
// // //check that there's nothing in the mount folder
// // filesInDir, err := ioutil.ReadDir(mountPoint)
// // if err != nil {
// // t.Fatalf("had an error reading the directory: %v", err)
// // }
// //
// // if len(filesInDir) != 0 {
// // t.Fatal("there shouldn't be anything here")
// // }
// //
// // secondMountPoint, err := ioutil.TempDir("", "swarm-test")
// // log.Debug("swarmfs cli test", "2nd mount point at", secondMountPoint)
// // if err != nil {
// // t.Fatal(err)
// // }
// // defer os.RemoveAll(secondMountPoint)
// //
// // log.Debug("swarmfs cli test: remounting at second mount point", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
// //
// // //remount, check files
// // newMount := runSwarm(t, []string{
// // fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
// // "fs",
// // "mount",
// // hash, // the latest hash
// // secondMountPoint,
// // }...)
// //
// // newMount.ExpectExit()
// // time.Sleep(1 * time.Second)
// //
// // filesInDir, err = ioutil.ReadDir(secondMountPoint)
// // if err != nil {
// // t.Fatal(err)
// // }
// //
// // if len(filesInDir) == 0 {
// // t.Fatal("there should be something here")
// // }
// //
// // log.Debug("swarmfs cli test: traversing file tree to see it matches previous mount")
// //
// // for _, file := range filesToAssert {
// // file.filePath = strings.Replace(file.filePath, mountPoint, secondMountPoint, -1)
// // fileBytes, err := ioutil.ReadFile(file.filePath)
// //
// // if err != nil {
// // t.Fatal(err)
// // }
// // if !bytes.Equal(fileBytes, bytes.NewBufferString(file.content).Bytes()) {
// // t.Fatal("this should be equal")
// // }
// // }
// //
// // log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
// //
// // unmountSec := runSwarm(t, []string{
// // fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
// // "fs",
// // "unmount",
// // secondMountPoint,
// // }...)
// //
// // _, matches = unmountSec.ExpectRegexp(hashRegexp)
// // unmountSec.ExpectExit()
// //
// // if matches[0] != hash {
// // t.Fatal("these should be equal - no changes made")
// // }
// }
hashRegexp := `[a-f\d]{64}` // func doUploadEmptyDir(t *testing.T, node *testNode) string {
// // create a tmp dir
// tmpDir, err := ioutil.TempDir("", "swarm-test")
// if err != nil {
// t.Fatal(err)
// }
// defer os.RemoveAll(tmpDir)
flags := []string{ // hashRegexp := `[a-f\d]{64}`
"--bzzapi", node.URL,
"--recursive",
"up",
tmpDir}
log.Info("swarmfs cli test: uploading dir with 'swarm up'") // flags := []string{
up := runSwarm(t, flags...) // "--bzzapi", node.URL,
_, matches := up.ExpectRegexp(hashRegexp) // "--recursive",
up.ExpectExit() // "up",
hash := matches[0] // tmpDir}
log.Info("swarmfs cli test: dir uploaded", "hash", hash)
return hash
}
func createDirInDir(createInDir string, dirToCreate string) (string, error) { // log.Info("swarmfs cli test: uploading dir with 'swarm up'")
fullpath := filepath.Join(createInDir, dirToCreate) // up := runSwarm(t, flags...)
err := os.MkdirAll(fullpath, 0777) // _, matches := up.ExpectRegexp(hashRegexp)
if err != nil { // up.ExpectExit()
return "", err // hash := matches[0]
} // log.Info("swarmfs cli test: dir uploaded", "hash", hash)
return fullpath, nil // return hash
} // }
func createTestFileInPath(dir, filename, content string) (*testFile, error) { // func createDirInDir(createInDir string, dirToCreate string) (string, error) {
tFile := &testFile{} // fullpath := filepath.Join(createInDir, dirToCreate)
filePath := filepath.Join(dir, filename) // err := os.MkdirAll(fullpath, 0777)
if file, err := os.Create(filePath); err == nil { // if err != nil {
tFile.content = content // return "", err
tFile.filePath = filePath // }
// return fullpath, nil
// }
_, err = io.WriteString(file, content) // func createTestFileInPath(dir, filename, content string) (*testFile, error) {
if err != nil { // tFile := &testFile{}
return nil, err // filePath := filepath.Join(dir, filename)
} // if file, err := os.Create(filePath); err == nil {
file.Close() // tFile.content = content
} // tFile.filePath = filePath
return tFile, nil // _, err = io.WriteString(file, content)
} // if err != nil {
// return nil, err
// }
// file.Close()
// }
// return tFile, nil
// }

View file

@ -63,29 +63,29 @@ func TestSwarmNetwork(t *testing.T) {
options *testSwarmNetworkOptions options *testSwarmNetworkOptions
disabled bool disabled bool
}{ }{
{ // {
name: "10_nodes", // name: "10_nodes",
steps: []testSwarmNetworkStep{ // steps: []testSwarmNetworkStep{
{ // {
nodeCount: 10, // nodeCount: 10,
}, // },
}, // },
options: &testSwarmNetworkOptions{ // options: &testSwarmNetworkOptions{
Timeout: 45 * time.Second, // Timeout: 45 * time.Second,
}, // },
}, // },
{ // {
name: "10_nodes_skip_check", // name: "10_nodes_skip_check",
steps: []testSwarmNetworkStep{ // steps: []testSwarmNetworkStep{
{ // {
nodeCount: 10, // nodeCount: 10,
}, // },
}, // },
options: &testSwarmNetworkOptions{ // options: &testSwarmNetworkOptions{
Timeout: 45 * time.Second, // Timeout: 45 * time.Second,
SkipCheck: true, // SkipCheck: true,
}, // },
}, // },
{ {
name: "50_nodes", name: "50_nodes",
steps: []testSwarmNetworkStep{ steps: []testSwarmNetworkStep{

View file

@ -1362,6 +1362,9 @@ func worker(id int, jobs <-chan Job, rpcs map[enode.ID]*rpc.Client, pubkeys map[
} }
func TestNetwork(t *testing.T) { func TestNetwork(t *testing.T) {
if !*longrunning {
t.Skip("run with --longrunning flag to run extensive network tests")
}
t.Run("16/1000/4/sim", testNetwork) t.Run("16/1000/4/sim", testNetwork)
} }