Mirrors
/
go-benchmarks
mirror of https://github.com/golang/benchmarks.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cmd/bent: add -X flag for benchmark/library-modification experiments 

Normally, for no-surprises reproducibility, bent always starts
the build directory from scratch.  This change adds a flag that
turns off that clean-start behavior, so that local edits to
build/*/go.mod for various benchmarks are not reverted.  For
example, if a change to Go breaks a library used by a benchmark,
replace directives in go.mod allow use of a modified library.

Change-Id: Ida8d3776eae081cc4fae63a0489cbb0579b7c5a8
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/470935
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: David Chase <drchase@google.com>
Reviewed-by: Austin Clements <austin@google.com>
pull/8/head
David Chase 2023-02-23 17:43:15 -05:00
parent 354f4ce956
commit 1c7bc14bf2
1 changed files with 51 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
cmd/bent/bent.go
View File

@ -79,6 +79,7 @@ var explicitAll counterFlag // Include "-a" on "go test -c" test build ; repeati
var shuffle = 2 // Dimensionality of (build) shuffling; 0 = none, 1 = per-benchmark, configuration ordering, 2 = bench, config pairs, 3 = across repetitions.
var haveRsync = true
var reportBuildTime = true
var experiment = false // Don't reset go.mod, for testing purposes
//go:embed scripts/*
var scripts embed.FS
@ -149,6 +150,7 @@ func main() {
flag.BoolVar(&test, "T", test, "run tests instead of benchmarks")
flag.BoolVar(&wikiTable, "W", wikiTable, "print benchmark info for a wiki table")
flag.BoolVar(&experiment, "X", experiment, "for experimental changes to 3rd party software, do not reset build/*/go.mod")
flag.BoolVar(&reportBuildTime, "report-build-time", reportBuildTime, "report build real/CPU time as benchmark results")
@ -535,47 +537,56 @@ results will also appear in 'bench'.
fmt.Printf("Couldn't create build subdirectory %s, error=%v", bench.BuildDir, err)
os.Exit(2)
}
getFiles := true
goDotMod := path.Join(bench.BuildDir, "go.mod")
if _, err := os.Stat(goDotMod); err == nil {
os.Remove(goDotMod) // always want a fresh go.mod
if !experiment {
os.Remove(goDotMod) // always want a fresh go.mod
} else {
getFiles = false
}
}
cmd := exec.Command("go", "mod", "init", "build")
cmd.Env = DefaultEnv()
cmd.Dir = bench.BuildDir
if getFiles {
cmd := exec.Command("go", "mod", "init", "build")
cmd.Env = DefaultEnv()
cmd.Dir = bench.BuildDir
if verbose > 0 {
fmt.Println(asCommandLine(dirs.wd, cmd))
} else {
fmt.Print(".")
}
_, err := cmd.Output()
if err != nil {
ee := err.(*exec.ExitError)
fmt.Printf("There was an error running 'go mod init', stderr = %s", ee.Stderr)
os.Exit(2)
}
if verbose > 0 {
fmt.Println(asCommandLine(dirs.wd, cmd))
} else {
fmt.Print(".")
}
_, err := cmd.Output()
if err != nil {
ee := err.(*exec.ExitError)
fmt.Printf("There was an error running 'go mod init', stderr = %s", ee.Stderr)
os.Exit(2)
}
cmd = exec.Command("go", "get", "-d", "-t", "-v", bench.Repo+bench.Version)
cmd.Env = DefaultEnv()
cmd.Dir = bench.BuildDir
cmd = exec.Command("go", "get", "-d", "-t", "-v", bench.Repo+bench.Version)
cmd.Env = DefaultEnv()
cmd.Dir = bench.BuildDir
if !bench.NotSandboxed { // Do this so that OS-dependent dependencies are done correctly.
cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
}
cmd.Env = replaceEnvs(cmd.Env, sliceExpandEnv(bench.GcEnv, cmd.Env))
if verbose > 0 {
fmt.Println(asCommandLine(dirs.wd, cmd))
} else {
fmt.Print(".")
}
_, err = cmd.Output()
if err != nil {
ee := err.(*exec.ExitError)
s := fmt.Sprintf("There was an error running 'go get', stderr = %s", ee.Stderr)
fmt.Println(s + "DISABLING benchmark " + bench.Name)
getAndBuildFailures = append(getAndBuildFailures, s+"("+bench.Name+")\n")
todo.Benchmarks[i].Disabled = true
continue
if !bench.NotSandboxed { // Do this so that OS-dependent dependencies are done correctly.
cmd.Env = replaceEnv(cmd.Env, "GOOS", "linux")
}
cmd.Env = replaceEnvs(cmd.Env, sliceExpandEnv(bench.GcEnv, cmd.Env))
if verbose > 0 {
fmt.Println(asCommandLine(dirs.wd, cmd))
} else {
fmt.Print(".")
}
_, err = cmd.Output()
if err != nil {
ee := err.(*exec.ExitError)
s := fmt.Sprintf("There was an error running 'go get', stderr = %s", ee.Stderr)
fmt.Println(s + "DISABLING benchmark " + bench.Name)
getAndBuildFailures = append(getAndBuildFailures, s+"("+bench.Name+")\n")
todo.Benchmarks[i].Disabled = true
continue
}
}
needSandbox = !bench.NotSandboxed || needSandbox
@ -672,14 +683,11 @@ results will also appear in 'bench'.
}
}
// Prebuild the library for this configuration unless -a=1
if explicitAll != 1 {
if needSandbox {
buildLibrary(true)
}
if needNotSandbox {
buildLibrary(false)
}
if needSandbox {
buildLibrary(true)
}
if needNotSandbox {
buildLibrary(false)
}
todo.Configurations[ci] = config
if config.Disabled {