Add some automation for doing periodic mass rebuilds of Fedora packag…

…es (#864)

These mass rebuilds will help test the snapshot builds and catch
regressions in real-world projects.  The GitHub workflows are
currently configured to start a new mass rebuild once per month
and then when the rebuild is complete, it will create any issue
reporting any regressions since the last build.

The goal is to keep the rebuilder as simple as possible, so it will
only rebuild packages that successfully built in the last rebuild, and
it makes no attempt to work around packages that hard code gcc as their
compiler.  As a result, some of the 'passing' builds may actually use
gcc instead of clang, but this is an acceptable trade-off to keep
the rebuild process simple.

Co-authored-by: Konrad Kleine <konrad.kleine@posteo.de>

.github/actions/prepare-python/action.yml

@@ -7,11 +7,16 @@ inputs:
     description: "Where this project was checked out"
     required: false
     default: "."
+  use-system-python:
+    description: "Set to true if you want to use the python installation from the OS"
+    required: false
+    default: false
 runs:
   using: "composite"
   steps:
     - name: "Setup python"
       uses: actions/setup-python@v5
+      if: "${{ inputs.use-system-python != 'true' }}"
       with:
         python-version: 3.12
         cache: 'pip'
@@ -24,4 +29,14 @@ runs:
 
     - name: "Install python dependencies for project"
       shell: bash
-      run: pip install -r ${{ inputs.checkout-path }}/requirements.txt
+      run: |
+        pip install -r ${{ inputs.checkout-path }}/requirements.txt
+
+    # The dnf module cannot be installed by pip, so it's only possible to use
+    # it when using the system pyhton interpreter.
+    - name: Install DNF module
+      if: "${{ inputs.use-system-python == 'true' }}"
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-dnf

.github/workflows/build-reproducer-container.yml

@@ -0,0 +1,27 @@
+name: "Build Reproducer Container"
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  build-reproducer-container:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Container
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install podman
+          podman build -t ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer -f Containerfile scripts/
+          podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
+          podman push ghcr.io/$GITHUB_REPOSITORY_OWNER/llvm-snapshots-reproducer

.github/workflows/mass-rebuild-bisect.yml

@@ -0,0 +1,40 @@
+name: Mass Rebuild Bisect
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      pkg:
+        description: "The name of the Fedora package build that you want to bisect."
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      pkg:
+        description: "The name of the Fedora package build that you want to bisect."
+        required: true
+        type: string
+
+jobs:
+  mass-rebuild-bisect:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    container:
+      image: "ghcr.io/${{ github.repository_owner }}/llvm-snapshots-reproducer"
+    steps:
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          max-size: 8G
+          key: bisect
+      - working-directory: /root/llvm-project/
+        run: |
+          git fetch origin
+          bash bisect.sh ${{ inputs.pkg }}
+
+      - if: always()
+        working-directory: /root/llvm-project
+        run: |
+          git bisect log

.github/workflows/mass-rebuild-reporter.yml

@@ -0,0 +1,115 @@
+name: "Mass Rebuild Reporter"
+
+on:
+  schedule:
+    # Hourly at minute 40, e.g. 2024-12-18 00:40:00
+    - cron: "40 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  check-for-rebuild:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    permissions:
+      issues: write
+    container:
+      image: "registry.fedoraproject.org/fedora:41"
+    outputs:
+      regressions: ${{ steps.regressions.outputs.REGRESSIONS }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts/rebuilder.py
+          sparse-checkout-cone-mode: false
+
+
+      - name: Check for last report
+        uses: actions/github-script@v7
+        id: last-report
+        with:
+          result-encoding: string
+          script: |
+            const issues = await github.rest.search.issuesAndPullRequests({
+              q: "label:mass-rebuild+is:issue",
+              sort: "created",
+              order: "desc",
+              per_page: 1
+            });
+
+            console.log(issues)
+            if (issues.data.total_count == 0)
+              return 0;
+            const issue = issues.data.items[0];
+            console.log(issue);
+            return issue.created_at
+
+      - name: Check if a new rebuild has completed
+        id: new-rebuild
+        run: |
+          sudo dnf install -y python3-dnf python3-copr
+          if python3 scripts/rebuilder.py rebuild-in-progress; then
+            echo "completed=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          last_rebuild=$(date +%s -d "${{ steps.last-report.outputs.result }}")
+          current_snapshot=$(date +%s -d "$(python3 scripts/rebuilder.py get-snapshot-date)")
+
+          echo "last_rebuild: $last_rebuild current_snapshot: $current_snapshot"
+
+          if [ $last_rebuild -gt $current_snapshot ]; then
+            echo "completed=false" >> $GITHUB_OUTPUT
+          else
+            echo "completed=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Collect Regressions
+        if: steps.new-rebuild.outputs.completed == 'true'
+        id: regressions
+        run: |
+          python3 scripts/rebuilder.py get-regressions --start-date ${{ steps.last-report.outputs.result }} > regressions
+          echo "REGRESSIONS=$(cat regressions)" >> $GITHUB_OUTPUT
+
+      - name: Create Report
+        if: steps.new-rebuild.outputs.completed == 'true'
+        uses: actions/github-script@v7
+        env:
+          REGRESSIONS: ${{ steps.regressions.outputs.REGRESSIONS }}
+        with:
+          script: |
+            var fs = require('fs');
+            const regressions = await JSON.parse(fs.readFileSync('./regressions'));
+            comment = "During the last mass rebuild, some packages failed:\n";
+            console.log(regressions);
+            if (regressions.length == 0)
+              return;
+            regressions.forEach(function(value){
+              comment = comment.concat('\n', value.name);
+              comment = comment.concat(': ', value.url);
+            });
+            console.log(comment);
+            const issue = await github.rest.issues.create({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               title: "Mass Rebuild Report",
+               labels: ['mass-rebuild'],
+               body: comment
+            });
+            console.log(issue);
+
+  bisect-failures:
+    if: github.repository_owner == 'fedora-llvm-team'
+    needs:
+      - check-for-rebuild
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.check-for-rebuild.outputs.regressions) }}
+    uses: ./.github/workflows/mass-rebuild-bisect.yml
+    with:
+      pkg: ${{ matrix.name }}

.github/workflows/mass-rebuild-runner.yml

@@ -0,0 +1,37 @@
+name: "Mass Rebuild Runner"
+
+on:
+  schedule:
+    # Run on the first of every month.
+    - cron: 30 1 1 * *
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  start-rebuild:
+    if: github.repository_owner == 'fedora-llvm-team'
+    runs-on: ubuntu-24.04
+    container:
+      image: "registry.fedoraproject.org/fedora:41"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            scripts/rebuilder.py
+          sparse-checkout-cone-mode: false
+
+      - name: Setup Copr config file
+        env:
+          # You need to have those secrets in your repo.
+          # See also: https://copr.fedorainfracloud.org/api/.
+          COPR_CONFIG_FILE: ${{ secrets.COPR_CONFIG }}
+        run: |
+          mkdir -p ~/.config
+          printf "$COPR_CONFIG_FILE" > ~/.config/copr
+
+      - name: Start rebuild
+        run: |
+          sudo dnf install -y python3-dnf python3-copr
+          python3 scripts/rebuilder.py rebuild

.github/workflows/python-format-and-tests.yml

@@ -19,6 +19,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/prepare-python
+        with:
+          use-system-python: true
       - name: Run pytest with coverage
         shell: bash -e {0}
         env:

Containerfile

@@ -0,0 +1,19 @@
+FROM registry.fedoraproject.org/fedora:41
+
+ENV LLVM_SYSROOT=/opt/llvm \
+    AR=llvm-ar \
+    RANLIB=llvm-ranlib
+
+RUN dnf -y copr enable tstellar/fedora-clang-default-cc
+
+RUN dnf -y install jq cmake ninja-build git binutils-devel clang fedora-clang-default-cc rpmbuild ccache
+
+WORKDIR /root
+
+RUN git clone https://github.com/llvm/llvm-project
+
+WORKDIR /root/llvm-project
+
+ADD bisect.sh git-bisect-script.sh .
+
+COPY --from=ghcr.io/llvm/ci-ubuntu-22.04:1734145213 $LLVM_SYSROOT $LLVM_SYSROOT

README.adoc

@@ -193,6 +193,38 @@ $ cd llvm
 $ make snapshot-rpm
 ----
 
+= README
+:icons: font
+
 You might need to install missing dependencies. The build process itself probably takes quite some time.
 
 You're going to find `results/YYYYMMDD/snapshot-rpm.log` with logging everything from this makefile target.
+
+== Mass Rebuilds ==
+
+This repository uses GitHub Actions to periodically perform rebuilds of selected
+Fedora packages.  Once a mass rebuild is complete there is also automation
+that will create a new issue with the results of the rebuild.
+
+The rebuild process will attempt to automatically bisect the failures to a specific upstream
+LLVM commit.
+
+The rebuild can be started manually using the rebuilder.py script in
+`.github/workflows/`
+
+[source,console]
+---
+$ python3 rebuilder.py rebuild
+---
+
+You can also view the regression report once the rebuild is complete using
+the same script.
+
+[source,console]
+---
+$ python3 rebuilder.py get-regressions --start-date=<yyyy-mm-dd>
+---
+
+The start date should be the day the rebuild was started (In reality
+it can be any date between when the last rebuild ended and the
+new rebuild began).

pytest.ini

@@ -11,7 +11,7 @@ addopts: --doctest-modules --durations=0 -s
 
 ; Directories to search for tests when no files or directories are given on the
 ; command line
-testpaths: snapshot_manager
+testpaths: snapshot_manager scripts
 
 ; Each line specifies a pattern for warnings.filterwarnings. Processed after
 ; -W/--pythonwarnings.

scripts/bisect.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -ex
+
+function get_clang_commit {
+  buildid=$1
+  pkg=$2
+
+  curl "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/fedora-41-clang-20/fedora-41-x86_64/0$buildid-$pkg/root.log.gz" | gunzip |  grep -o 'clang[[:space:]]\+x86_64[[:space:]]\+[0-9a-g~pre.]\+' | cut -d 'g' -f 3
+}
+
+
+pkg_or_buildid=$1
+
+if echo $pkg_or_buildid | grep '^[0-9]\+'; then
+  buildid=$pkg_or_buildid
+  read -r pkg last_success_id <<<$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r '[.builds.latest.source_package.name,.builds.latest_succeeded.id] |  join(" ")')
+else
+  pkg=$pkg_or_buildid
+fi
+
+read -r buildid last_success_id <<<$(curl -X 'GET' \
+  "https://copr.fedorainfracloud.org/api_3/package/?ownername=%40fedora-llvm-team&projectname=fedora-41-clang-20&packagename=$pkg&with_latest_build=true&with_latest_succeeded_build=true" \
+  -H 'accept: application/json' | jq -r '[.builds.latest.id,.builds.latest_succeeded.id] | join(" ")' )
+
+
+good_commit=llvmorg-20-init
+bad_commit=origin/main
+
+good_commit=$(get_clang_commit $last_success_id $pkg)
+bad_commit=$(get_clang_commit $buildid $pkg)
+
+srpm_url=$(curl -X 'GET' "https://copr.fedorainfracloud.org/api_3/build/$buildid" -H 'accept: application/json' | jq -r .source_package.url)
+curl -O -L $srpm_url
+srpm_name=$(basename $srpm_url)
+
+dnf builddep -y $srpm_name
+
+
+git bisect start
+git bisect good $good_commit
+git bisect bad $bad_commit
+
+cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD=Native -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang
+
+git bisect run ./git-bisect-script.sh $srpm_name

scripts/git-bisect-script.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+srpm_name=$1
+
+ninja -C build install-clang install-clang-resource-headers install-LLVMgold install-llvm-ar install-llvm-ranlib
+
+rpmbuild -rb $srpm_name