Add optional --image-file-dir to store loaded files elsewhere (apache…

…#45564) While backorting the "pull_request_target" removal to v2-10-test branches it turned out that there is not enough disk space on Public runner to load all 5 images and keep the file dump at the same time in the same filesystem. This PR allows to choose where the load/save files will be stored and in the github runner environment we store the files in "/mnt" wnich is a separate folder with 40GB free.
PolideaInternal · Jan 11, 2025 · 6628049 · 6628049
1 parent 4c01f7d
commit 6628049
Show file tree

Hide file tree

Showing 18 changed files with 246 additions and 120 deletions.
diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml
@@ -46,17 +46,22 @@ runs:
       with:
         use-uv: ${{ inputs.use-uv }}
       id: breeze
+    - name: Check free space
+      run: df -H
+      shell: bash
+    - name: Make /mnt/ directory writeable
+      run: sudo chown -R ${USER} /mnt
+      shell: bash
     - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
       uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
         key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
-        path: "/tmp/"
+        path: "/mnt/"
     - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
       env:
         PLATFORM: ${{ inputs.platform }}
         PYTHON: ${{ inputs.python }}
         IMAGE_TYPE: ${{ inputs.image-type }}
       run: >
-        breeze ${IMAGE_TYPE}-image load
-        --platform ${PLATFORM} --python ${PYTHON}
+        breeze ${IMAGE_TYPE}-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt"
       shell: bash
diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml
@@ -35,16 +35,22 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Check free space
+      run: df -H
+      shell: bash
+    - name: Make /mnt/ directory writeable
+      run: sudo chown -R ${USER} /mnt
+      shell: bash
     - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
       uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
       with:
         key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
-        path: "/tmp/"
+        path: "/mnt/"
       if: contains(inputs.python-versions-list-as-string, inputs.python)
     - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"
       env:
         PLATFORM: ${{ inputs.platform }}
         PYTHON: ${{ inputs.python }}
-      run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}"
+      run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt/"
       shell: bash
       if: contains(inputs.python-versions-list-as-string, inputs.python)
diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
@@ -175,16 +175,22 @@ jobs:
           PUSH: ${{ inputs.push-image }}
           VERBOSE: "true"
           PLATFORM: ${{ inputs.platform }}
+      - name: Check free space
+        run: df -H
+        shell: bash
+      - name: Make /mnt/ directory writeable
+        run: sudo chown -R ${USER} /mnt
+        shell: bash
       - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         env:
           PLATFORM: ${{ inputs.platform }}
-        run: breeze ci-image save --platform "${PLATFORM}"
+        run: breeze ci-image save --platform "${PLATFORM}" --image-file-dir "/mnt"
         if: inputs.upload-image-artifact == 'true'
       - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
         with:
           key: ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}
-          path: "/tmp/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
+          path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
           if-no-files-found: 'error'
           retention-days: '2'
         if: inputs.upload-image-artifact == 'true'

diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
@@ -283,17 +283,23 @@ jobs:
         if: inputs.build-provider-packages != 'true'
       - name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         run: breeze prod-image verify
+      - name: Check free space
+        run: df -H
+        shell: bash
+      - name: Make /mnt/ directory writeable
+        run: sudo chown -R ${USER} /mnt
+        shell: bash
       - name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         env:
           PLATFORM: ${{ inputs.platform }}
         run: >
-          breeze prod-image save --platform "${PLATFORM}"
+          breeze prod-image save --platform "${PLATFORM}" --image-file-dir "/mnt"
         if: inputs.upload-image-artifact == 'true'
       - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
         with:
           key: prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}
-          path: "/tmp/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
+          path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
           if-no-files-found: 'error'
           retention-days: '2'
         if: inputs.upload-image-artifact == 'true'