Migrate non-Linux C++ tests to GHA actions

PiperOrigin-RevId: 506683178

.github/actions/bazel-docker/action.yml

@@ -11,7 +11,7 @@ inputs:
     description: "The docker image to use"
     type: string
   bazel-cache:
-    required: false
+    required: true
     description: >
       A unique path for the Bazel cache.  This will trigger the generation
       of a BAZEL_CACHE environment variable inside the container that provides
@@ -31,35 +31,25 @@ runs:
   steps:
     - name: Authenticate
       id: auth
-      uses: ./.github/actions/internal/docker-auth
+      uses: ./.github/actions/internal/gcloud-auth
       with:
         credentials: ${{ inputs.credentials }}
 
+    - name: Setup Runner
+      uses: ./.github/actions/internal/setup-runner
+
+    - name: Setup Bazel
+      id: bazel
+      uses: ./.github/actions/internal/bazel-setup
+      with:
+        credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
+        bazel-cache: ${{ inputs.bazel-cache }}
+
     - name: Validate inputs
-      if: ${{ inputs.bash && inputs.bazel}}
+      if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
       shell: bash
       run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
 
-    - name: Initialize Bazel flags
-      shell: bash
-      run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV
-
-    - name: Configure Bazel caching
-      shell: bash
-      # Skip bazel cache for local act runs due to issue with credential files
-      # and nested docker images
-      if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
-      run: >
-        echo "BAZEL_FLAGS=$BAZEL_FLAGS
-        --google_credentials=/workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
-        --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
-
-    - name: Configure Bazel cache writing
-      shell: bash
-      # External PRs should never write to our caches.
-      if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }}
-      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV
-
     - name: Run Bash Docker
       uses: ./.github/actions/internal/docker-run
       if: ${{ inputs.bash }}
@@ -73,4 +63,4 @@ runs:
       if: ${{ !inputs.bash }}
       with:
         image: ${{ inputs.image }}
-        command: ${{ inputs.bazel }} $BAZEL_FLAGS
+        command: ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}

.github/actions/bazel/action.yml

@@ -0,0 +1,75 @@
+name: 'Docker Bazel Run'
+description: 'Run a Bazel-based docker image for Protobuf CI testing'
+inputs:
+  credentials:
+    required: true
+    description: The GCP credentials to use for reading the docker image
+    type: string
+  bazel-cache:
+    required: true
+    description: >
+      A unique path for the Bazel cache.  This will trigger the generation
+      of a BAZEL_CACHE environment variable inside the container that provides
+      the appropriate flags for any bazel command.
+    type: string
+  version:
+    required: false
+    description: A pinned Bazel version to use
+    default: '5.1.1'
+    type: string
+  bazel:
+    required: false
+    description: The Bazel command to run
+    type: string
+  bash:
+    required: false
+    description: >
+      A bash command to run.  $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be
+      available to use for bazel runs.
+    type: string
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Authenticate
+      id: auth
+      uses: ./.github/actions/internal/gcloud-auth
+      with:
+        credentials: ${{ inputs.credentials }}
+
+    - name: Setup Runner
+      uses: ./.github/actions/internal/setup-runner
+
+    - name: Setup Bazel
+      id: bazel
+      uses: ./.github/actions/internal/bazel-setup
+      with:
+        credentials-file: ${{ steps.auth.outputs.credentials-file }}
+        bazel-cache: ${{ inputs.bazel-cache }}
+
+    - name: Validate inputs
+      if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
+      shell: bash
+      run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
+
+    - name: Output Bazel version
+      env:
+        USE_BAZEL_VERSION: ${{ inputs.version }}
+      shell: bash
+      run: bazelisk version
+
+    - name: Run Bash
+      env:
+        USE_BAZEL_VERSION: ${{ inputs.version }}
+      if: ${{ inputs.bash }}
+      run: ${{ inputs.bash }}
+      shell: bash
+
+    - name: Run Bazel
+      env:
+        USE_BAZEL_VERSION: ${{ inputs.version }}
+      if: ${{ !inputs.bash }}
+      run: >
+        bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
+        ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}
+      shell: bash

.github/actions/docker/action.yml

@@ -17,6 +17,9 @@ inputs:
 runs:
   using: 'composite'
   steps:
+    - name: Setup Runner
+      uses: ./.github/actions/internal/setup-runner
+
     - name: Update stale files using Bazel
       uses: ./.github/actions/bazel-docker
       with:

.github/actions/internal/bazel-setup/action.yml

@@ -0,0 +1,59 @@
+name: Setup Bazel
+description: Setup a Bazel environment for Protobuf CI testing
+inputs:
+  credentials-file:
+    required: true
+    description: The GCP credentials file to use for caching
+    type: string
+  bazel-cache:
+    required: true
+    description: A unique path for the Bazel cache.
+    type: string
+
+outputs:
+  bazel-flags:
+    description: Bazel flags that should be sent to all Bazel invocations
+    value: ${{ steps.output.outputs.bazel-flags }}
+  bazel-startup-flags:
+    description: Bazel startup flags that should be sent to all Bazel invocations
+    value: ${{ steps.output.outputs.bazel-startup-flags }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Initialize Windows startup flags
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV
+
+    - name: Initialize Bazel flags
+      shell: bash
+      run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV
+
+    - name: Initialize Windows-specific Bazel flags
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV
+
+    - name: Configure Bazel caching
+      # Skip bazel cache for local act runs due to issue with credential files
+      # and nested docker images
+      if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
+      shell: bash
+      run: >
+        echo "BAZEL_FLAGS=$BAZEL_FLAGS
+        --google_credentials='${{ inputs.credentials-file }}'
+        --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
+
+    - name: Configure Bazel cache writing
+      # External runs should never write to our caches.
+      if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }}
+      shell: bash
+      run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV
+
+    - name: Output Bazel flags
+      id: output
+      shell: bash
+      run: |
+        echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT
+        echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT

.github/actions/internal/docker-run/action.yml

@@ -22,6 +22,10 @@ inputs:
 runs:
   using: 'composite'
   steps:
+    - name: Authenticate for GAR use
+      shell: bash
+      run: gcloud auth configure-docker -q us-docker.pkg.dev
+
     - name: Setup QEMU for possible emulation
       uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
 

.github/actions/internal/gcloud-auth/action.yml

@@ -1,9 +1,9 @@
-name: 'Authenticate Docker'
+name: 'Authenticate for GCP'
 description: 'Authenticate a workflow for Protobuf CI testing'
 inputs:
   credentials:
     required: true
-    description: "The GCP credentials to use for reading the docker image"
+    description: "The GCP credentials to use for GCP"
     type: string
 
 outputs:
@@ -24,6 +24,3 @@ runs:
     - name: Use gcloud CLI
       shell: bash
       run: gcloud info
-    - name: Authenticate for GAR use
-      shell: bash
-      run: gcloud auth configure-docker -q us-docker.pkg.dev

.github/actions/internal/setup-runner/action.yml

@@ -0,0 +1,17 @@
+name: Setup CI Runner
+# TODO(b/267357823) Consider moving this to it's own repository and include
+# a call to actions/checkout.
+description: Setup any platform-specific adjustments we need to make for CI
+runs:
+  using: 'composite'
+  steps:
+    - name: Fix Windows line breaks
+      if: runner.os == 'Windows'
+      shell: bash
+      run: find . -type f -print0 | xargs -0 d2u 2>/dev/null
+
+    - name: Install bazelrc files
+      shell: bash
+      run: |
+        cp ci/*.bazelrc .
+        cp -f ${{ runner.os }}.bazelrc .bazelrc

.github/workflows/test_cpp.yml

@@ -88,8 +88,33 @@ jobs:
           submodules: recursive
           ref: ${{ inputs.safe-checkout }}
       - name: Run tests
-        uses: ./.github/actions/non-bazel-docker
+        uses: ./.github/actions/docker
         with:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: ${{ matrix.command }}
+
+  non-linux:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        include:
+          - name: MacOS
+            os: macos-12
+            bazel: //src/...
+          - name: Windows
+            os: windows-2019
+            bazel: //src/...  @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance
+    name: ${{ matrix.name }} Bazel
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          ref: ${{ inputs.safe-checkout }}
+      - name: Run tests
+        uses: ./.github/actions/bazel
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel: test ${{ matrix.bazel }}
+          bazel-cache: cpp_${{ matrix.os }}

.github/workflows/test_php.yml

@@ -43,7 +43,7 @@ jobs:
           submodules: recursive
           ref: ${{ inputs.safe-checkout }}
       - name: Run tests
-        uses: ./.github/actions/non-bazel-docker
+        uses: ./.github/actions/docker
         with:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}

ci/Linux.bazelrc

@@ -0,0 +1,3 @@
+import common.bazelrc
+
+build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14

ci/Windows.bazelrc

@@ -0,0 +1,2 @@
+import common.bazelrc
+

ci/common.bazelrc

@@ -0,0 +1,33 @@
+build:dbg --compilation_mode=dbg
+
+build:opt --compilation_mode=opt
+
+build:san-common --config=dbg --strip=never --copt=-O0 --copt=-fno-omit-frame-pointer
+
+build:asan --config=san-common --copt=-fsanitize=address --linkopt=-fsanitize=address
+build:asan --copt=-DADDRESS_SANITIZER=1
+# ASAN hits ODR violations with shared linkage due to rules_proto.
+build:asan --dynamic_mode=off
+
+build:msan --config=san-common --copt=-fsanitize=memory --linkopt=-fsanitize=memory
+build:msan --copt=-fsanitize-memory-track-origins
+build:msan --copt=-fsanitize-memory-use-after-dtor
+build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
+build:msan --copt=-DMEMORY_SANITIZER=1
+
+# Use our instrumented LLVM libc++ in Kokoro.
+build:kokoro-msan --config=msan
+build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib
+build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
+build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
+
+
+build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread
+build:tsan --copt=-DTHREAD_SANITIZER=1
+
+build:ubsan --config=san-common --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined
+build:ubsan --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
+build:ubsan --copt=-DUNDEFINED_SANITIZER=1
+# Workaround for the fact that Bazel links with $CC, not $CXX
+# https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748
+build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr

ci/macOS.bazelrc

@@ -0,0 +1,3 @@
+import common.bazelrc
+
+build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14

src/google/protobuf/io/zero_copy_stream_unittest.cc

@@ -731,9 +731,11 @@ TEST_F(IoTest, StringIo) {
 
 // Verifies that outputs up to kint32max can be created.
 TEST_F(IoTest, LargeOutput) {
-  // Filter out this test on 32-bit architectures and tsan builds.
+  // Filter out this test on 32-bit architectures and builds where our test
+  // infrastructure can't handle it.
   if(sizeof(void*) < 8) return;
-#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER)
+#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && \
+    !defined(_MSC_VER)
   std::string str;
   StringOutputStream output(&str);
   void* unused_data;