diff --git a/.github/workflows/nightly-bnb.yml b/.github/workflows/nightly-bnb.yml
index b6f2143fac..573d9a37ed 100644
--- a/.github/workflows/nightly-bnb.yml
+++ b/.github/workflows/nightly-bnb.yml
@@ -85,6 +85,13 @@ jobs:
           pip install -e . --no-deps
           pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
           mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone
+          if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
+            cd transformers-clone
+            transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
+            echo "Checking out tag for Transformers version: v$transformers_version"
+            git fetch --tags
+            git checkout tags/v$transformers_version
+            cd .. 
 
       - name: Run core GPU tests on multi-gpu
         if: always()