ci(homebrew): run tests in separate step

.github/workflows/ci-homebrew.yml

@@ -60,6 +60,8 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
 
       - name: Select Xcode 16.2
         if: matrix.os_name == 'macos' && matrix.os_version == '14'
@@ -194,6 +196,68 @@ jobs:
           validate: true
       - run: echo "::remove-matcher owner=gcc-strip3::"
 
+      - name: Run Homebrew tests
+        if: matrix.release != true
+        env:
+          ACTION_TESTPATH: ${{ steps.test.outputs.testpath }}
+        run: |
+          set -euo pipefail
+
+          homebrew_temp="${RUNNER_TEMP}/sunshine-homebrew-test"
+          homebrew_testpath="${homebrew_temp}/sunshine/test"
+          mkdir -p "${homebrew_testpath}/tests"
+          cp "${ACTION_TESTPATH}/coverage-buildpath.txt" "${homebrew_testpath}/coverage-buildpath.txt"
+
+          brew install --only-dependencies --include-test lizardbyte/homebrew/sunshine
+
+          formula_prefix="$(brew --prefix lizardbyte/homebrew/sunshine)"
+          test_binary="${formula_prefix}/bin/test_sunshine"
+          test_runtime="${formula_prefix}/libexec/tests"
+          if [[ ! -x "${test_binary}" ]]; then
+            echo "::error::Homebrew test binary was not installed at ${test_binary}"
+            exit 1
+          fi
+          if [[ ! -d "${test_runtime}" ]]; then
+            echo "::error::Homebrew test fixtures were not installed at ${test_runtime}"
+            exit 1
+          fi
+
+          cd "${test_runtime}"
+          set +e
+          if [[ "${RUNNER_OS}" == "macOS" ]]; then
+            LLVM_PROFILE_FILE="${homebrew_testpath}/sunshine-%p.profraw" \
+            "${test_binary}" \
+                --gtest_color=yes \
+                --gtest_output="xml:${homebrew_testpath}/tests/test_results.xml"
+            test_status=$?
+            coverage_file="coverage.lcov"
+          else
+            # Homebrew's test sandbox cannot write gcov intermediates into the installed Cellar tree.
+            coverage_runtime="$(< "${homebrew_testpath}/coverage-buildpath.txt")"
+            mkdir -p "${coverage_runtime}"
+            cp -R "${test_runtime}/coverage/." "${coverage_runtime}/"
+            printf '%s\n' "${coverage_runtime}" > "${homebrew_testpath}/coverage-buildpath.txt"
+            gcov_prefix_strip="$(< "${test_runtime}/gcov-prefix-strip.txt")"
+            GCOV_PREFIX="${coverage_runtime}" \
+              GCOV_PREFIX_STRIP="${gcov_prefix_strip}" \
+              "${test_binary}" \
+                --gtest_color=yes \
+                --gtest_output="xml:${homebrew_testpath}/tests/test_results.xml"
+            test_status=$?
+            coverage_file="coverage.xml"
+          fi
+          set -e
+
+          HOMEBREW_TEMP="${homebrew_temp}" \
+            HOMEBREW_TEST_ARTIFACTS_DIR="${homebrew_testpath}" \
+            brew test lizardbyte/homebrew/sunshine
+
+          mkdir -p "${ACTION_TESTPATH}/tests"
+          cp "${homebrew_testpath}/${coverage_file}" "${ACTION_TESTPATH}/${coverage_file}"
+          cp "${homebrew_testpath}/tests/test_results.xml" "${ACTION_TESTPATH}/tests/test_results.xml"
+
+          exit "${test_status}"
+
       - name: Upload coverage artifact
         if: >-
           always() &&

packaging/sunshine.rb

@@ -5,13 +5,16 @@ class Sunshine < Formula
 
   CUDA_VERSION = "13.1".freeze
   CUDA_FORMULA = "cuda@#{CUDA_VERSION}".freeze
+  COVERAGE_BUILDPATH_FILE = "coverage-buildpath.txt".freeze
   COVERAGE_LCOV = "coverage.lcov".freeze
   COVERAGE_PROFDATA = "coverage.profdata".freeze
   COVERAGE_XML = "coverage.xml".freeze
+  GCOV_PREFIX_STRIP_FILE = "gcov-prefix-strip.txt".freeze
   GCC_VERSION = "14".freeze
   GCC_FORMULA = "gcc@#{GCC_VERSION}".freeze
   LLVM_PROFILE_FILE_ENV = "LLVM_PROFILE_FILE".freeze
   TEST_BINARY = "test_sunshine".freeze
+  TEST_RESULTS_XML = "tests/test_results.xml".freeze
   IS_UPSTREAM_REPO = ENV.fetch("GITHUB_REPOSITORY", "") == "LizardByte/Sunshine"
 
   desc "@PROJECT_DESCRIPTION@"
@@ -47,6 +50,9 @@ class Sunshine < Formula
   option "with-static-boost", "Enable static link of Boost libraries"
   option "without-static-boost", "Disable static link of Boost libraries" # default option
 
+  # Keep coverage instrumentation in the test binary after test-bot rebuilds the formula from its bottle.
+  skip_clean "bin/#{TEST_BINARY}" if IS_UPSTREAM_REPO
+
   depends_on "cmake" => :build
   depends_on "doxygen" => :build if build.with? "docs"
   depends_on "graphviz" => :build if build.with? "docs"
@@ -180,7 +186,17 @@ def base_cmake_args
   def add_test_args(args)
     if IS_UPSTREAM_REPO
       args << "-DBUILD_TESTS=ON"
-      args << "-DSUNSHINE_LLVM_COVERAGE=ON" if OS.mac?
+      test_runtime = opt_libexec/"tests"
+      args << "-DSUNSHINE_TEST_SOURCE_DIR=#{test_runtime}"
+      args << "-DSUNSHINE_TEST_RUNTIME_DIR=#{test_runtime}"
+      coverage_arg = if OS.mac?
+        "-DSUNSHINE_LLVM_COVERAGE=ON"
+      else
+        # gcovr writes intermediate files next to the mapped sources, so use Homebrew's writable temp tree.
+        coverage_runtime = HOMEBREW_TEMP/"coverage"
+        "-DSUNSHINE_TEST_GCOV_ROOT=#{coverage_runtime}"
+      end
+      args << coverage_arg
       ohai "Building tests: enabled"
     else
       args << "-DBUILD_TESTS=OFF"
@@ -260,7 +276,7 @@ def ensure_artifact_exists(path)
 
   def run_test_suite(artifact_dir)
     mkdir_p artifact_dir/"tests"
-    test_results = artifact_dir/"tests/test_results.xml"
+    test_results = artifact_dir/TEST_RESULTS_XML
 
     if OS.mac?
       with_llvm_profile_file(artifact_dir) do
@@ -345,7 +361,7 @@ def generate_llvm_coverage_report(artifact_dir, coverage_buildpath, coverage_rep
 
   def generate_gcov_coverage_report(coverage_report, coverage_buildpath)
     cd "#{coverage_buildpath}/build" do
-      system "gcovr", ".",
+      system "gcovr", "tests/CMakeFiles/#{TEST_BINARY}.dir/__/src",
         "-r", "../src",
         *coverage_gcov_options,
         *coverage_common_options(coverage_report)
@@ -355,10 +371,9 @@ def generate_gcov_coverage_report(coverage_report, coverage_buildpath)
   end
 
   def coverage_source_prefixes(coverage_buildpath)
-    paths = [
-      coverage_buildpath.to_s,
-      Pathname.new(coverage_buildpath.to_s).realpath.to_s,
-    ]
+    coverage_buildpath = Pathname.new(coverage_buildpath.to_s)
+    paths = [coverage_buildpath.to_s]
+    paths << coverage_buildpath.realpath.to_s if coverage_buildpath.exist?
     paths.uniq.map { |path| "#{path}/src/" }
   end
 
@@ -405,8 +420,9 @@ def collect_test_artifacts
     return unless IS_UPSTREAM_REPO
     return unless artifact_dir
 
-    run_test_suite artifact_dir
-    generate_coverage_report artifact_dir, buildpath
+    coverage_buildpath = OS.mac? ? buildpath.realpath : HOMEBREW_TEMP/"coverage"
+    mkdir_p artifact_dir
+    (artifact_dir/COVERAGE_BUILDPATH_FILE).write coverage_buildpath.to_s
   end
 
   def build_cmake_args
@@ -428,7 +444,29 @@ def build_and_install_project
   end
 
   def install_platform_specific_files
-    bin.install "build/tests/#{TEST_BINARY}" if IS_UPSTREAM_REPO
+    if IS_UPSTREAM_REPO
+      bin.install "build/tests/#{TEST_BINARY}"
+
+      test_runtime = libexec/"tests"
+      %w[docs src src_assets test_assets].each do |directory|
+        test_runtime.install "build/tests/#{directory}"
+      end
+      test_runtime.install "sunshine.png"
+      (test_runtime/"tests/unit").install "tests/unit/test_video.cpp"
+
+      if OS.linux?
+        coverage_runtime = test_runtime/"coverage"
+        coverage_runtime.install "src"
+        (test_runtime/GCOV_PREFIX_STRIP_FILE).write "#{buildpath.each_filename.count}\n"
+        coverage_notes = buildpath.glob("build/**/*.gcno")
+        odie "No gcov notes were created" if coverage_notes.empty?
+
+        coverage_notes.each do |coverage_note|
+          relative_note = coverage_note.relative_path_from(buildpath)
+          (coverage_runtime/relative_note.dirname).install coverage_note
+        end
+      end
+    end
 
     # codesign the binary on intel macs
     system "codesign", "-s", "-", "--force", "--deep", bin/"sunshine" if OS.mac? && Hardware::CPU.intel?
@@ -484,8 +522,27 @@ def caveats
     if IS_UPSTREAM_REPO
       artifact_dir = release_homebrew_testpath
       if artifact_dir
-        assert_path_exists artifact_dir/"tests/test_results.xml"
-        assert_path_exists coverage_report_path(artifact_dir)
+        coverage_buildpath = artifact_dir/COVERAGE_BUILDPATH_FILE
+        test_runtime = opt_libexec/"tests"
+        assert_path_exists coverage_buildpath
+        assert_path_exists bin/TEST_BINARY
+        assert_path_exists test_runtime/"docs/getting_started.md"
+        assert_path_exists test_runtime/"src/config.cpp"
+        assert_path_exists test_runtime/"src_assets/common/assets/web/public/assets/locale/en.json"
+        assert_path_exists test_runtime/"test_assets/web/images/logo-sunshine.svg"
+        assert_path_exists test_runtime/"tests/unit/test_video.cpp"
+        if OS.linux?
+          assert_path_exists test_runtime/"coverage/src/config.cpp"
+          assert_path_exists test_runtime/GCOV_PREFIX_STRIP_FILE
+          source_notes = test_runtime.glob("coverage/build/tests/CMakeFiles/#{TEST_BINARY}.dir/__/src/**/*.gcno")
+          assert source_notes.any?, "No installed source gcov notes were found"
+        end
+
+        if (artifact_dir/TEST_RESULTS_XML).exist?
+          assert_path_exists artifact_dir/TEST_RESULTS_XML
+          generate_coverage_report artifact_dir, coverage_buildpath.read.strip
+          assert_path_exists coverage_report_path(artifact_dir)
+        end
       elsif ENV.fetch("HOMEBREW_BOTTLE_BUILD", "false") != "true"
         run_test_suite testpath
         generate_coverage_report testpath, ENV.fetch("HOMEBREW_BUILDPATH", "")
@@ -526,6 +583,19 @@ def caveats
         end_of_record
       LCOV
       assert_equal expected_lcov, lcov_for_source_files(lcov, testpath)
+
+      missing_buildpath = testpath/"missing-buildpath"
+      missing_path_lcov = <<~LCOV
+        SF:#{missing_buildpath}/src/missing.cpp
+        DA:1,1
+        end_of_record
+      LCOV
+      expected_missing_path_lcov = <<~LCOV
+        SF:src/missing.cpp
+        DA:1,1
+        end_of_record
+      LCOV
+      assert_equal expected_missing_path_lcov, lcov_for_source_files(missing_path_lcov, missing_buildpath)
     end
   end
 end

tests/CMakeLists.txt

@@ -34,6 +34,14 @@ else()
     set(CMAKE_C_FLAGS "-fprofile-arcs -ftest-coverage -ggdb -O0")
 endif()
 
+set(SUNSHINE_TEST_GCOV_ROOT
+        ""
+        CACHE PATH "Relocated source and build root used to process gcov data")
+if(SUNSHINE_TEST_GCOV_ROOT AND CMAKE_C_COMPILER_ID STREQUAL "GNU")
+    add_compile_options(
+            "-fprofile-prefix-map=${CMAKE_SOURCE_DIR}=${SUNSHINE_TEST_GCOV_ROOT}")
+endif()
+
 # Link the libgcov that belongs to the selected compiler. Asking GCC directly
 # avoids selecting another installed GCC version through platform-specific
 # library directory names (for example, the GCC required by CUDA on Arch Linux).
@@ -69,19 +77,25 @@ elseif (NOT APPLE)
 endif ()
 
 set(TEST_DEFINITIONS)  # list will be appended as needed
+set(SUNSHINE_TEST_SOURCE_DIR
+        "${CMAKE_SOURCE_DIR}"
+        CACHE PATH "Source fixture directory used by tests")
+set(SUNSHINE_TEST_RUNTIME_DIR
+        "${CMAKE_CURRENT_BINARY_DIR}"
+        CACHE PATH "Writable runtime fixture directory used by tests")
 
 # this indicates we're building tests in case sunshine needs to adjust some code or add private tests
 list(APPEND TEST_DEFINITIONS SUNSHINE_TESTS)
-list(APPEND TEST_DEFINITIONS SUNSHINE_SOURCE_DIR="${CMAKE_SOURCE_DIR}")
-list(APPEND TEST_DEFINITIONS SUNSHINE_TEST_BIN_DIR="${CMAKE_CURRENT_BINARY_DIR}")
+list(APPEND TEST_DEFINITIONS SUNSHINE_SOURCE_DIR="${SUNSHINE_TEST_SOURCE_DIR}")
+list(APPEND TEST_DEFINITIONS SUNSHINE_TEST_BIN_DIR="${SUNSHINE_TEST_RUNTIME_DIR}")
 if(SUNSHINE_ENABLE_TRAY)
     list(APPEND TEST_DEFINITIONS TRAY_ENABLE_TEST_HOOKS)
 endif()
 
 # Override SUNSHINE_ASSETS_DIR to use a writable temp directory for tests
 # Remove the existing definition from SUNSHINE_DEFINITIONS to avoid redefinition error
 list(FILTER SUNSHINE_DEFINITIONS EXCLUDE REGEX "^SUNSHINE_ASSETS_DIR=")
-list(APPEND TEST_DEFINITIONS SUNSHINE_ASSETS_DIR="${CMAKE_CURRENT_BINARY_DIR}/test_assets")
+list(APPEND TEST_DEFINITIONS SUNSHINE_ASSETS_DIR="${SUNSHINE_TEST_RUNTIME_DIR}/test_assets")
 
 if(NOT WIN32)
     find_package(Udev 255)  # we need 255+ for udevadm verify