all: treat all files as binary, but check in .bat with CRLF

This is a followup to CL 96495.

It should be simpler and more robust to achieve .bat files having
CRLF line endings by treating it as a binary file, like all other
files, and checking it in with the desired CRLF line endings.

A test is used to check the entire Go tree, short of directories
starting with "." and named "testdata", for any .bat files that
have anything other than strict CRLF line endings. This will help
catch any accidental modifications to existing .bat files or check
ins of new .bat files.

Importantly, this is compatible with how Gerrit serves .tar.gz files,
making it so that CRLF line endings are preserved.

The Go project is supported on many different environments, some of
which may have limited git implementations available, or none at all.
Relying on fewer git features and special rules makes it easier to
have confidence in the exact content of all files. Additionally, Go
development started in Subversion, moved to Perforce, then Mercurial,
and now uses Git.¹ Reducing its reliance on git-specific features will
help if there will be another transition in the project's future.

There are only 5 .bat files in the entire Go source tree, so a new one
being added is a rare event, and we prefer to do things in Go instead.
We still have the option of improving the experience for developers by
adding a pre-commit converter for .bat files to the git-codereview tool.

¹ https://groups.google.com/d/msg/golang-dev/sckirqOWepg/YmyT7dWJiocJ

Fixes #39391.
For #37791.

Change-Id: I6e202216322872f0307ac96f1b8d3f57cb901e6b
Reviewed-on: https://go-review.googlesource.com/c/go/+/236437
Reviewed-by: Bryan C. Mills <bcmills@google.com>
changes/37/236437/7
Dmitri Shuralyov 2020-06-04 00:35:09 -04:00
parent 5c6b2b14db
commit ee379d2b08
7 changed files with 377 additions and 340 deletions

20
.gitattributes vendored
View File

@ -1,16 +1,16 @@
# Treat all files in the Go repo as binary, with no git magic updating # Treat all files in the Go repo as binary, with no git magic updating
# line endings. Windows users contributing to Go will need to use a # line endings. This produces predictable results in different environments.
# modern version of git and editors capable of LF line endings. #
# Windows users contributing to Go will need to use a modern version
# of git and editors capable of LF line endings.
#
# Windows .bat files are known to have multiple bugs when run with LF
# endings, and so they are checked in with CRLF endings, with a test
# in test/winbatch.go to catch problems. (See golang.org/issue/37791.)
# #
# We'll prevent accidental CRLF line endings from entering the repo # We'll prevent accidental CRLF line endings from entering the repo
# via the git-review gofmt checks. # via the git-codereview gofmt checks and tests.
# #
# See golang.org/issue/9281 # See golang.org/issue/9281.
* -text * -text
# The only exception is Windows files that must absolutely be CRLF or
# might not work. Batch files are known to have multiple bugs when run
# with LF endings. See golang.org/issue/37791 for more information.
*.bat text eol=crlf

View File

@ -1,27 +1,27 @@
:: Copyright 2012 The Go Authors. All rights reserved. :: Copyright 2012 The Go Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style :: Use of this source code is governed by a BSD-style
:: license that can be found in the LICENSE file. :: license that can be found in the LICENSE file.
@echo off @echo off
setlocal setlocal
if exist make.bat goto ok if exist make.bat goto ok
echo all.bat must be run from go\src echo all.bat must be run from go\src
:: cannot exit: would kill parent command interpreter :: cannot exit: would kill parent command interpreter
goto end goto end
:ok :ok
set OLDPATH=%PATH% set OLDPATH=%PATH%
call make.bat --no-banner --no-local call make.bat --no-banner --no-local
if %GOBUILDFAIL%==1 goto end if %GOBUILDFAIL%==1 goto end
call run.bat --no-rebuild --no-local call run.bat --no-rebuild --no-local
if %GOBUILDFAIL%==1 goto end if %GOBUILDFAIL%==1 goto end
:: we must restore %PATH% before running "dist banner" so that the latter :: we must restore %PATH% before running "dist banner" so that the latter
:: can get the original %PATH% and give suggestion to add %GOROOT%/bin :: can get the original %PATH% and give suggestion to add %GOROOT%/bin
:: to %PATH% if necessary. :: to %PATH% if necessary.
set PATH=%OLDPATH% set PATH=%OLDPATH%
"%GOTOOLDIR%/dist" banner "%GOTOOLDIR%/dist" banner
:end :end
if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL% if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL%

View File

@ -1,32 +1,32 @@
:: Copyright 2012 The Go Authors. All rights reserved. :: Copyright 2012 The Go Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style :: Use of this source code is governed by a BSD-style
:: license that can be found in the LICENSE file. :: license that can be found in the LICENSE file.
@echo off @echo off
setlocal setlocal
set GOBUILDFAIL=0 set GOBUILDFAIL=0
go tool dist env -w -p >env.bat go tool dist env -w -p >env.bat
if errorlevel 1 goto fail if errorlevel 1 goto fail
call env.bat call env.bat
del env.bat del env.bat
echo. echo.
if exist %GOTOOLDIR%\dist.exe goto distok if exist %GOTOOLDIR%\dist.exe goto distok
echo cannot find %GOTOOLDIR%\dist; nothing to clean echo cannot find %GOTOOLDIR%\dist; nothing to clean
goto fail goto fail
:distok :distok
"%GOBIN%\go" clean -i std "%GOBIN%\go" clean -i std
"%GOBIN%\go" tool dist clean "%GOBIN%\go" tool dist clean
"%GOBIN%\go" clean -i cmd "%GOBIN%\go" clean -i cmd
goto end goto end
:fail :fail
set GOBUILDFAIL=1 set GOBUILDFAIL=1
:end :end
if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL% if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL%

View File

@ -1,153 +1,152 @@
:: Copyright 2012 The Go Authors. All rights reserved. :: Copyright 2012 The Go Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style :: Use of this source code is governed by a BSD-style
:: license that can be found in the LICENSE file. :: license that can be found in the LICENSE file.
:: Environment variables that control make.bat: :: Environment variables that control make.bat:
:: ::
:: GOROOT_FINAL: The expected final Go root, baked into binaries. :: GOROOT_FINAL: The expected final Go root, baked into binaries.
:: The default is the location of the Go tree during the build. :: The default is the location of the Go tree during the build.
:: ::
:: GOHOSTARCH: The architecture for host tools (compilers and :: GOHOSTARCH: The architecture for host tools (compilers and
:: binaries). Binaries of this type must be executable on the current :: binaries). Binaries of this type must be executable on the current
:: system, so the only common reason to set this is to set :: system, so the only common reason to set this is to set
:: GOHOSTARCH=386 on an amd64 machine. :: GOHOSTARCH=386 on an amd64 machine.
:: ::
:: GOARCH: The target architecture for installed packages and tools. :: GOARCH: The target architecture for installed packages and tools.
:: ::
:: GOOS: The target operating system for installed packages and tools. :: GOOS: The target operating system for installed packages and tools.
:: ::
:: GO_GCFLAGS: Additional go tool compile arguments to use when :: GO_GCFLAGS: Additional go tool compile arguments to use when
:: building the packages and commands. :: building the packages and commands.
:: ::
:: GO_LDFLAGS: Additional go tool link arguments to use when :: GO_LDFLAGS: Additional go tool link arguments to use when
:: building the commands. :: building the commands.
:: ::
:: CGO_ENABLED: Controls cgo usage during the build. Set it to 1 :: CGO_ENABLED: Controls cgo usage during the build. Set it to 1
:: to include all cgo related files, .c and .go file with "cgo" :: to include all cgo related files, .c and .go file with "cgo"
:: build directive, in the build. Set it to 0 to ignore them. :: build directive, in the build. Set it to 0 to ignore them.
:: ::
:: CC: Command line to run to compile C code for GOHOSTARCH. :: CC: Command line to run to compile C code for GOHOSTARCH.
:: Default is "gcc". :: Default is "gcc".
:: ::
:: CC_FOR_TARGET: Command line to run compile C code for GOARCH. :: CC_FOR_TARGET: Command line to run compile C code for GOARCH.
:: This is used by cgo. Default is CC. :: This is used by cgo. Default is CC.
:: ::
:: FC: Command line to run to compile Fortran code. :: FC: Command line to run to compile Fortran code.
:: This is used by cgo. Default is "gfortran". :: This is used by cgo. Default is "gfortran".
@echo off @echo off
:: Keep environment variables within this script :: Keep environment variables within this script
:: unless invoked with --no-local. :: unless invoked with --no-local.
if x%1==x--no-local goto nolocal if x%1==x--no-local goto nolocal
if x%2==x--no-local goto nolocal if x%2==x--no-local goto nolocal
if x%3==x--no-local goto nolocal if x%3==x--no-local goto nolocal
if x%4==x--no-local goto nolocal if x%4==x--no-local goto nolocal
setlocal setlocal
:nolocal :nolocal
set GOENV=off set GOENV=off
set GOBUILDFAIL=0 set GOBUILDFAIL=0
set GOFLAGS= set GOFLAGS=
set GO111MODULE= set GO111MODULE=
if exist make.bat goto ok if exist make.bat goto ok
echo Must run make.bat from Go src directory. echo Must run make.bat from Go src directory.
goto fail goto fail
:ok :ok
:: Clean old generated file that will cause problems in the build. :: Clean old generated file that will cause problems in the build.
del /F ".\pkg\runtime\runtime_defs.go" 2>NUL del /F ".\pkg\runtime\runtime_defs.go" 2>NUL
:: Set GOROOT for build. :: Set GOROOT for build.
cd .. cd ..
set GOROOT_TEMP=%CD% set GOROOT_TEMP=%CD%
set GOROOT= set GOROOT=
cd src cd src
set vflag= set vflag=
if x%1==x-v set vflag=-v if x%1==x-v set vflag=-v
if x%2==x-v set vflag=-v if x%2==x-v set vflag=-v
if x%3==x-v set vflag=-v if x%3==x-v set vflag=-v
if x%4==x-v set vflag=-v if x%4==x-v set vflag=-v
if not exist ..\bin\tool mkdir ..\bin\tool if not exist ..\bin\tool mkdir ..\bin\tool
:: Calculating GOROOT_BOOTSTRAP :: Calculating GOROOT_BOOTSTRAP
if not "x%GOROOT_BOOTSTRAP%"=="x" goto bootstrapset if not "x%GOROOT_BOOTSTRAP%"=="x" goto bootstrapset
for /f "tokens=*" %%g in ('where go 2^>nul') do ( for /f "tokens=*" %%g in ('where go 2^>nul') do (
if "x%GOROOT_BOOTSTRAP%"=="x" ( if "x%GOROOT_BOOTSTRAP%"=="x" (
for /f "tokens=*" %%i in ('%%g env GOROOT 2^>nul') do ( for /f "tokens=*" %%i in ('%%g env GOROOT 2^>nul') do (
if /I not %%i==%GOROOT_TEMP% ( if /I not %%i==%GOROOT_TEMP% (
set GOROOT_BOOTSTRAP=%%i set GOROOT_BOOTSTRAP=%%i
) )
) )
) )
) )
if "x%GOROOT_BOOTSTRAP%"=="x" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\Go1.4 if "x%GOROOT_BOOTSTRAP%"=="x" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\Go1.4
:bootstrapset :bootstrapset
if not exist "%GOROOT_BOOTSTRAP%\bin\go.exe" goto bootstrapfail if not exist "%GOROOT_BOOTSTRAP%\bin\go.exe" goto bootstrapfail
set GOROOT=%GOROOT_TEMP% set GOROOT=%GOROOT_TEMP%
set GOROOT_TEMP= set GOROOT_TEMP=
echo Building Go cmd/dist using %GOROOT_BOOTSTRAP% echo Building Go cmd/dist using %GOROOT_BOOTSTRAP%
if x%vflag==x-v echo cmd/dist if x%vflag==x-v echo cmd/dist
setlocal setlocal
set GOROOT=%GOROOT_BOOTSTRAP% set GOROOT=%GOROOT_BOOTSTRAP%
set GOOS= set GOOS=
set GOARCH= set GOARCH=
set GOBIN= set GOBIN=
set GO111MODULE=off set GO111MODULE=off
"%GOROOT_BOOTSTRAP%\bin\go.exe" build -o cmd\dist\dist.exe .\cmd\dist "%GOROOT_BOOTSTRAP%\bin\go.exe" build -o cmd\dist\dist.exe .\cmd\dist
endlocal endlocal
if errorlevel 1 goto fail if errorlevel 1 goto fail
.\cmd\dist\dist.exe env -w -p >env.bat .\cmd\dist\dist.exe env -w -p >env.bat
if errorlevel 1 goto fail if errorlevel 1 goto fail
call env.bat call env.bat
del env.bat del env.bat
if x%vflag==x-v echo. if x%vflag==x-v echo.
if x%1==x--dist-tool goto copydist if x%1==x--dist-tool goto copydist
if x%2==x--dist-tool goto copydist if x%2==x--dist-tool goto copydist
if x%3==x--dist-tool goto copydist if x%3==x--dist-tool goto copydist
if x%4==x--dist-tool goto copydist if x%4==x--dist-tool goto copydist
set buildall=-a set buildall=-a
if x%1==x--no-clean set buildall= if x%1==x--no-clean set buildall=
if x%2==x--no-clean set buildall= if x%2==x--no-clean set buildall=
if x%3==x--no-clean set buildall= if x%3==x--no-clean set buildall=
if x%4==x--no-clean set buildall= if x%4==x--no-clean set buildall=
if x%1==x--no-banner set buildall=%buildall% --no-banner if x%1==x--no-banner set buildall=%buildall% --no-banner
if x%2==x--no-banner set buildall=%buildall% --no-banner if x%2==x--no-banner set buildall=%buildall% --no-banner
if x%3==x--no-banner set buildall=%buildall% --no-banner if x%3==x--no-banner set buildall=%buildall% --no-banner
if x%4==x--no-banner set buildall=%buildall% --no-banner if x%4==x--no-banner set buildall=%buildall% --no-banner
:: Run dist bootstrap to complete make.bash. :: Run dist bootstrap to complete make.bash.
:: Bootstrap installs a proper cmd/dist, built with the new toolchain. :: Bootstrap installs a proper cmd/dist, built with the new toolchain.
:: Throw ours, built with Go 1.4, away after bootstrap. :: Throw ours, built with Go 1.4, away after bootstrap.
.\cmd\dist\dist.exe bootstrap %vflag% %buildall% .\cmd\dist\dist.exe bootstrap %vflag% %buildall%
if errorlevel 1 goto fail if errorlevel 1 goto fail
del .\cmd\dist\dist.exe del .\cmd\dist\dist.exe
goto end goto end
:: DO NOT ADD ANY NEW CODE HERE. :: DO NOT ADD ANY NEW CODE HERE.
:: The bootstrap+del above are the final step of make.bat. :: The bootstrap+del above are the final step of make.bat.
:: If something must be added, add it to cmd/dist's cmdbootstrap, :: If something must be added, add it to cmd/dist's cmdbootstrap,
:: to avoid needing three copies in three different shell languages :: to avoid needing three copies in three different shell languages
:: (make.bash, make.bat, make.rc). :: (make.bash, make.bat, make.rc).
:copydist :copydist
mkdir "%GOTOOLDIR%" 2>NUL mkdir "%GOTOOLDIR%" 2>NUL
copy cmd\dist\dist.exe "%GOTOOLDIR%\" copy cmd\dist\dist.exe "%GOTOOLDIR%\"
goto end goto end
:bootstrapfail :bootstrapfail
echo ERROR: Cannot find %GOROOT_BOOTSTRAP%\bin\go.exe echo ERROR: Cannot find %GOROOT_BOOTSTRAP%\bin\go.exe
echo Set GOROOT_BOOTSTRAP to a working Go tree ^>= Go 1.4. echo Set GOROOT_BOOTSTRAP to a working Go tree ^>= Go 1.4.
:fail :fail
set GOBUILDFAIL=1 set GOBUILDFAIL=1
if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL% if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL%
:end :end

View File

@ -1,51 +1,51 @@
:: Copyright 2013 The Go Authors. All rights reserved. :: Copyright 2013 The Go Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style :: Use of this source code is governed by a BSD-style
:: license that can be found in the LICENSE file. :: license that can be found in the LICENSE file.
:: race.bash tests the standard library under the race detector. :: race.bash tests the standard library under the race detector.
:: https://golang.org/doc/articles/race_detector.html :: https://golang.org/doc/articles/race_detector.html
@echo off @echo off
setlocal setlocal
if exist make.bat goto ok if exist make.bat goto ok
echo race.bat must be run from go\src echo race.bat must be run from go\src
:: cannot exit: would kill parent command interpreter :: cannot exit: would kill parent command interpreter
goto end goto end
:ok :ok
set GOROOT=%CD%\.. set GOROOT=%CD%\..
call make.bat --dist-tool >NUL call make.bat --dist-tool >NUL
if errorlevel 1 goto fail if errorlevel 1 goto fail
.\cmd\dist\dist.exe env -w -p >env.bat .\cmd\dist\dist.exe env -w -p >env.bat
if errorlevel 1 goto fail if errorlevel 1 goto fail
call env.bat call env.bat
del env.bat del env.bat
if %GOHOSTARCH% == amd64 goto continue if %GOHOSTARCH% == amd64 goto continue
echo Race detector is only supported on windows/amd64. echo Race detector is only supported on windows/amd64.
goto fail goto fail
:continue :continue
call make.bat --no-banner --no-local call make.bat --no-banner --no-local
if %GOBUILDFAIL%==1 goto end if %GOBUILDFAIL%==1 goto end
echo # go install -race std echo # go install -race std
go install -race std go install -race std
if errorlevel 1 goto fail if errorlevel 1 goto fail
go tool dist test -race go tool dist test -race
if errorlevel 1 goto fail if errorlevel 1 goto fail
goto succ goto succ
:fail :fail
set GOBUILDFAIL=1 set GOBUILDFAIL=1
echo Fail. echo Fail.
goto end goto end
:succ :succ
echo All tests passed. echo All tests passed.
:end :end
if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL% if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL%

View File

@ -1,59 +1,59 @@
:: Copyright 2012 The Go Authors. All rights reserved. :: Copyright 2012 The Go Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style :: Use of this source code is governed by a BSD-style
:: license that can be found in the LICENSE file. :: license that can be found in the LICENSE file.
@echo off @echo off
if exist ..\bin\go.exe goto ok if exist ..\bin\go.exe goto ok
echo Must run run.bat from Go src directory after installing cmd/go. echo Must run run.bat from Go src directory after installing cmd/go.
goto fail goto fail
:ok :ok
:: Keep environment variables within this script :: Keep environment variables within this script
:: unless invoked with --no-local. :: unless invoked with --no-local.
if x%1==x--no-local goto nolocal if x%1==x--no-local goto nolocal
if x%2==x--no-local goto nolocal if x%2==x--no-local goto nolocal
setlocal setlocal
:nolocal :nolocal
set GOBUILDFAIL=0 set GOBUILDFAIL=0
:: we disallow local import for non-local packages, if %GOROOT% happens :: we disallow local import for non-local packages, if %GOROOT% happens
:: to be under %GOPATH%, then some tests below will fail :: to be under %GOPATH%, then some tests below will fail
set GOPATH= set GOPATH=
:: Issue 14340: ignore GOBIN during all.bat. :: Issue 14340: ignore GOBIN during all.bat.
set GOBIN= set GOBIN=
set GOFLAGS= set GOFLAGS=
set GO111MODULE= set GO111MODULE=
rem TODO avoid rebuild if possible rem TODO avoid rebuild if possible
if x%1==x--no-rebuild goto norebuild if x%1==x--no-rebuild goto norebuild
echo ##### Building packages and commands. echo ##### Building packages and commands.
..\bin\go install -a -v std cmd ..\bin\go install -a -v std cmd
if errorlevel 1 goto fail if errorlevel 1 goto fail
echo. echo.
:norebuild :norebuild
:: we must unset GOROOT_FINAL before tests, because runtime/debug requires :: we must unset GOROOT_FINAL before tests, because runtime/debug requires
:: correct access to source code, so if we have GOROOT_FINAL in effect, :: correct access to source code, so if we have GOROOT_FINAL in effect,
:: at least runtime/debug test will fail. :: at least runtime/debug test will fail.
set GOROOT_FINAL= set GOROOT_FINAL=
:: get CGO_ENABLED :: get CGO_ENABLED
..\bin\go env > env.bat ..\bin\go env > env.bat
if errorlevel 1 goto fail if errorlevel 1 goto fail
call env.bat call env.bat
del env.bat del env.bat
echo. echo.
..\bin\go tool dist test ..\bin\go tool dist test
if errorlevel 1 goto fail if errorlevel 1 goto fail
echo. echo.
goto end goto end
:fail :fail
set GOBUILDFAIL=1 set GOBUILDFAIL=1
:end :end

View File

@ -4,8 +4,8 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// Check that batch files are maintained as CRLF files (consistent behaviour // Check that batch files are maintained as CRLF files (consistent
// on all operating systems). See https://github.com/golang/go/issues/37791 // behavior on all operating systems). See golang.org/issue/37791.
package main package main
@ -13,18 +13,56 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings"
) )
func main() { func main() {
batches, _ := filepath.Glob(runtime.GOROOT() + "/src/*.bat") // Ensure that the GOROOT/src/all.bat file exists and has strict CRLF line endings.
for _, bat := range batches { enforceBatchStrictCRLF(filepath.Join(runtime.GOROOT(), "src", "all.bat"))
body, _ := ioutil.ReadFile(bat)
if !bytes.Contains(body, []byte("\r\n")) { // Walk the entire Go repository source tree (without GOROOT/pkg),
fmt.Printf("Windows batch file %s does not contain CRLF line termination.\nTry running git checkout src/*.bat to fix this.\n", bat) // skipping directories that start with "." and named "testdata",
os.Exit(1) // and ensure all .bat files found have exact CRLF line endings.
err := filepath.Walk(runtime.GOROOT(), func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
} }
if fi.IsDir() && (strings.HasPrefix(fi.Name(), ".") || fi.Name() == "testdata") {
return filepath.SkipDir
}
if path == filepath.Join(runtime.GOROOT(), "pkg") {
// GOROOT/pkg is known to contain generated artifacts, not source code.
// Skip it to avoid false positives. (Also see golang.org/issue/37929.)
return filepath.SkipDir
}
if filepath.Ext(fi.Name()) == ".bat" {
enforceBatchStrictCRLF(path)
}
return nil
})
if err != nil {
log.Fatalln(err)
}
}
func enforceBatchStrictCRLF(path string) {
b, err := ioutil.ReadFile(path)
if err != nil {
log.Fatalln(err)
}
cr, lf := bytes.Count(b, []byte{13}), bytes.Count(b, []byte{10})
crlf := bytes.Count(b, []byte{13, 10})
if cr != crlf || lf != crlf {
if rel, err := filepath.Rel(runtime.GOROOT(), path); err == nil {
// Make the test failure more readable by showing a path relative to GOROOT.
path = rel
}
fmt.Printf("Windows batch file %s does not use strict CRLF line termination.\n", path)
fmt.Printf("Please convert it to CRLF before checking it in due to golang.org/issue/37791.\n")
os.Exit(1)
} }
} }