fix(ci): build linux x86_64 standard inside manylinux_2_28 for glibc 2.28+ compat (#9415)

Signed-off-by: Andrew Mello <andrew@88plug.com>
Co-authored-by: Alex Hancock <alex@alexhancock.com>
Co-authored-by: jh-block <255854896+jh-block@users.noreply.github.com>

.github/workflows/build-cli.yml

@@ -4,7 +4,7 @@
 #  - canary.yml
 #
 # Platform Build Strategy:
-#  - Linux standard: Uses native Ubuntu 22.04 runners to keep glibc compatibility with Ubuntu 22.04 LTS
+#  - Linux standard (x86_64 + aarch64): Builds inside manylinux_2_28 container for glibc 2.28+ compat
 #  - Linux Vulkan: Uses native Ubuntu 24.04 runners for newer Vulkan headers/tooling
 #  - Linux musl: Uses native Ubuntu 22.04 runners with reduced features for musl compatibility
 #  - macOS: Uses native macOS runners for each architecture
@@ -27,6 +27,7 @@ jobs:
   build-cli:
     name: Build CLI
     runs-on: ${{ matrix.build-on }}
+    container: ${{ matrix.container }}
     env:
       MACOSX_DEPLOYMENT_TARGET: "12.0"
     strategy:
@@ -37,11 +38,15 @@ jobs:
             architecture: x86_64
             target-suffix: unknown-linux-gnu
             build-on: ubuntu-22.04
+            # Pinned by digest for reproducible builds; bump explicitly when newer manylinux_2_28 images ship.
+            container: quay.io/pypa/manylinux_2_28_x86_64@sha256:441c35fdc6ee809ff9260894f8468ab4fea8c15dc880f8700a3f81b7922c1cda
             variant: standard
           - platform: linux
             architecture: aarch64
             target-suffix: unknown-linux-gnu
             build-on: ubuntu-22.04-arm
+            # Pinned by digest for reproducible builds; bump explicitly when newer manylinux_2_28 images ship.
+            container: quay.io/pypa/manylinux_2_28_aarch64@sha256:8b5f2b4e8c072ae5aefeb659f22c03e1ff46e6a82f154b6c904b106c87e65ff7
             variant: standard
           - platform: linux
             architecture: x86_64
@@ -97,8 +102,8 @@ jobs:
           sed -i.bak 's/^version = ".*"/version = "'${{ inputs.version }}'"/' Cargo.toml
           rm -f Cargo.toml.bak
 
-      - name: Install Linux build dependencies
-        if: matrix.platform == 'linux'
+      - name: Install Linux build dependencies (host runner)
+        if: matrix.platform == 'linux' && matrix.container == ''
         run: |
           sudo apt-get update
           sudo apt-get install -y \
@@ -119,11 +124,28 @@ jobs:
             sudo apt-get install -y musl-tools
           fi
 
+      - name: Install Linux build dependencies (manylinux container)
+        if: matrix.platform == 'linux' && matrix.container != ''
+        run: |
+          # perl-core provides FindBin, File::Compare, etc. that openssl-sys's
+          # vendored openssl build needs; in AlmaLinux 8 these aren't standalone packages.
+          # clang provides libclang.so for bindgen (used by llama-cpp-sys-2).
+          # Defensive: avoid actions/checkout falling back to a tarball download if base image changes.
+          dnf install -y --setopt=install_weak_deps=False \
+            openssl-devel \
+            dbus-devel \
+            libxcb-devel \
+            cmake \
+            perl-core \
+            clang \
+            git \
+            tar
+
       - name: Cache Cargo artifacts (Linux/macOS)
         if: matrix.platform != 'windows'
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4  # v2.9.1
         with:
-          key: ${{ matrix.architecture }}-${{ matrix.target-suffix }}-${{ matrix.build-on }}-native-macos-deployment-target-12
+          key: ${{ matrix.architecture }}-${{ matrix.target-suffix }}-${{ matrix.build-on }}-${{ matrix.container || 'native' }}-macos-deployment-target-12
 
       - name: Cache Cargo artifacts (Windows)
         if: matrix.platform == 'windows'
@@ -131,8 +153,8 @@ jobs:
         with:
           key: windows-msvc-cli-${{ matrix.variant }}
 
-      - name: Build CLI (Linux/macOS)
-        if: matrix.platform != 'windows'
+      - name: Build CLI (Linux/macOS host runner)
+        if: matrix.platform != 'windows' && matrix.container == ''
         env:
           RUST_LOG: debug
           RUST_BACKTRACE: 1
@@ -157,6 +179,27 @@ jobs:
             cargo build --release --target ${TARGET} -p goose-cli "${FEATURE_ARGS[@]}"
           fi
 
+      - name: Build CLI (manylinux container)
+        if: matrix.platform == 'linux' && matrix.container != ''
+        env:
+          RUST_BACKTRACE: 1
+        run: |
+          # Hermit's tool cache is host-runner-scoped; inside the container we
+          # bootstrap rustup directly and let rust-toolchain.toml pin the channel.
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+            | sh -s -- -y --default-toolchain none --profile minimal --no-modify-path
+          export PATH="$HOME/.cargo/bin:$PATH"
+          TARGET="${{ matrix.architecture }}-${{ matrix.target-suffix }}"
+          RUST_CHANNEL=$(grep '^channel' rust-toolchain.toml | cut -d'"' -f2)
+          if [ -z "$RUST_CHANNEL" ]; then
+            echo "Could not parse channel from rust-toolchain.toml" >&2
+            exit 1
+          fi
+          rustup toolchain install "$RUST_CHANNEL" --profile minimal \
+            --component rustc,cargo --target "$TARGET"
+          rustup show
+          cargo build --release --target "$TARGET" -p goose-cli
+
       - name: Setup Rust (Windows)
         if: matrix.platform == 'windows'
         shell: bash
@@ -215,7 +258,10 @@ jobs:
       - name: Package CLI (Linux/macOS)
         if: matrix.platform != 'windows'
         run: |
-          source ./bin/activate-hermit
+          # Hermit isn't installed in the manylinux container; tar is all this step needs.
+          if [ "${{ matrix.container }}" = '' ]; then
+            source ./bin/activate-hermit
+          fi
           export TARGET="${{ matrix.architecture }}-${{ matrix.target-suffix }}"
           export VARIANT_SUFFIX=""
           if [ "${{ matrix.variant }}" = "vulkan" ]; then