From 423afc368adb7c029f2ebec3ed904668b017aa83 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 14:06:46 -0700 Subject: [PATCH 01/28] fix: gradle and docker determinism working Signed-off-by: Matt Peterson --- .github/workflows/release-push-image.yaml | 38 +- .../generate-docker-artifact-baseline.sh | 265 +++++------ .../zxc-verify-docker-build-determinism.yaml | 445 +++++++++++++++++- .../zxc-verify-gradle-build-determinism.yaml | 84 ++-- server/docker/Dockerfile | 89 ++-- server/docker/docker-build.sh | 6 +- server/docker/repro-sources-list.sh | 99 ++++ 7 files changed, 788 insertions(+), 238 deletions(-) create mode 100755 server/docker/repro-sources-list.sh diff --git a/.github/workflows/release-push-image.yaml b/.github/workflows/release-push-image.yaml index bcba2e6b5..c0823be86 100644 --- a/.github/workflows/release-push-image.yaml +++ b/.github/workflows/release-push-image.yaml @@ -20,6 +20,7 @@ defaults: shell: bash permissions: + id-token: write contents: read packages: write @@ -29,21 +30,21 @@ env: REGISTRY: ghcr.io jobs: -# check-gradle: -# name: Gradle -# uses: ./.github/workflows/zxc-verify-gradle-build-determinism.yaml -# with: -# ref: ${{ github.event.inputs.ref || '' }} -# java-distribution: ${{ inputs.java-distribution || 'temurin' }} -# java-version: ${{ inputs.java-version || '21.0.4' }} - -# check-docker: -# name: Docker -# uses: ./.github/workflows/zxc-verify-docker-build-determinism.yaml -# with: -# ref: ${{ github.event.inputs.ref || '' }} -# java-distribution: ${{ inputs.java-distribution || 'temurin' }} -# java-version: ${{ inputs.java-version || '21.0.4' }} + check-gradle: + name: Gradle + uses: ./.github/workflows/zxc-verify-gradle-build-determinism.yaml + with: + ref: ${{ github.event.inputs.ref || '' }} + java-distribution: ${{ inputs.java-distribution || 'temurin' }} + java-version: ${{ inputs.java-version || '21.0.4' }} + + check-docker: + name: Docker + uses: ./.github/workflows/zxc-verify-docker-build-determinism.yaml + with: + ref: ${{ github.event.inputs.ref || '' }} + java-distribution: ${{ inputs.java-distribution || 'temurin' }} + java-version: ${{ inputs.java-version || '21.0.4' }} publish: runs-on: block-node-linux-medium @@ -87,6 +88,12 @@ jobs: VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Extract Source Date Epoch + id: extract_source_date_epoch + run: | + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" >> $GITHUB_ENV + - name: Build and push image uses: docker/build-push-action@1ca370b3a9802c92e886402e0dd88098a2533b12 # v6.4.1 with: @@ -99,6 +106,7 @@ jobs: tags: ${{ env.REGISTRY }}/${{ github.repository }}:${{ env.VERSION }} build-args: | VERSION=${{ env.VERSION }} + SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} build-contexts: | distributions=./server/build/distributions diff --git a/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh b/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh index b40966c62..4303a5f2a 100755 --- a/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh +++ b/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh @@ -2,7 +2,8 @@ set -o pipefail set +e -readonly DOCKER_IMAGE_NAME="server" +readonly DOCKER_IMAGE_NAME="hashgraph/hedera-block-node" +#readonly DOCKER_REGISTRY="ghcr.io" GROUP_ACTIVE="false" @@ -62,139 +63,141 @@ start_group "Configuring Environment" trap 'rm -rf "${TEMP_DIR}"' EXIT end_task "DONE (Path: ${TEMP_DIR})" -# start_task "Resolving the GITHUB_WORKSPACE path" -# # Ensure GITHUB_WORKSPACE is provided or default to the repository root -# if [[ -z "${GITHUB_WORKSPACE}" || ! -d "${GITHUB_WORKSPACE}" ]]; then -# GITHUB_WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" -# fi -# end_task "DONE (Path: ${GITHUB_WORKSPACE})" -# -# start_task "Resolving the GITHUB_OUTPUT path" -# # Ensure GITHUB_OUTPUT is provided or default to the repository root -# if [[ -z "${GITHUB_OUTPUT}" ]]; then -# GITHUB_OUTPUT="${TEMP_DIR}/workflow-output.txt" -# fi -# end_task "DONE (Path: ${GITHUB_OUTPUT})" -# -# start_task "Resolving the GITHUB_SHA hash" -# if [[ -z "${GITHUB_SHA}" ]]; then -# GITHUB_SHA="$(git rev-parse HEAD | tr -d '[:space:]')" || fail "ERROR (Exit Code: ${?})" "${?}" -# fi -# end_task "DONE (Commit: ${GITHUB_SHA})" -# -# start_task "Resolving the MANIFEST_PATH variable" -# if [[ -z "${MANIFEST_PATH}" ]]; then -# MANIFEST_PATH="${GITHUB_WORKSPACE}/.manifests/gradle" -# fi -# end_task "DONE (Path: ${MANIFEST_PATH})" -# -# start_task "Ensuring the MANIFEST_PATH location is present" -# if [[ ! -d "${MANIFEST_PATH}" ]]; then -# mkdir -p "${MANIFEST_PATH}" || fail "ERROR (Exit Code: ${?})" "${?}" -# fi -# end_task -# -# start_task "Resolving the SKOPEO_VERSION variable" -# if [[ -z "${SKOPEO_VERSION}" ]]; then -# SKOPEO_VERSION="v1.14.0" -# fi -# end_task "DONE (Version: ${SKOPEO_VERSION})" -# -# start_task "Resolving the SKOPEO_IMAGE_NAME variable" -# if [[ -z "${SKOPEO_IMAGE_NAME}" ]]; then -# SKOPEO_IMAGE_NAME="quay.io/skopeo/stable:${SKOPEO_VERSION}" -# fi -# end_task "DONE (Image Name: ${SKOPEO_IMAGE_NAME})" -# -# start_task "Checking for the DOCKER command" -# if command -v docker >/dev/null 2>&1; then -# DOCKER="$(command -v docker)" || fail "ERROR (Exit Code: ${?})" "${?}" -# export DOCKER -# else -# fail "ERROR (Exit Code: ${?})" "${?}" -# fi -# end_task "DONE (Found: ${DOCKER})" -# -# start_task "Resolving the Docker Client Configuration" -# SKOPEO_BIND_MOUNT="" -# SKOPEO_CREDENTIAL_OPTS="" -# DOCKER_CONFIG_DIR="${HOME}/.docker" -# if [[ -d "${DOCKER_CONFIG_DIR}" ]]; then -# SKOPEO_BIND_MOUNT="--volume ${DOCKER_CONFIG_DIR}:/tmp/docker" -# SKOPEO_CREDENTIAL_OPTS="--authfile /tmp/docker/config.json" -# fi -# export SKOPEO_BIND_MOUNT SKOPEO_CREDENTIAL_OPTS -# end_task "DONE" -# -# start_task "Checking for the SKOPEO command" -# if command -v skopeo >/dev/null 2>&1; then -# SKOPEO="$(command -v skopeo)" || fail "ERROR (Exit Code: ${?})" "${?}" -# export SKOPEO -# else -# ${DOCKER} pull "${SKOPEO_IMAGE_NAME}" >/dev/null 2>&1 || fail "ERROR (Exit Code: ${?})" "${?}" -# SKOPEO="${DOCKER} run ${SKOPEO_BIND_MOUNT} --rm --network host ${SKOPEO_IMAGE_NAME}" -# export SKOPEO -# fi -# end_task "DONE (Found: ${SKOPEO})" -# -# start_task "Checking for the JQ command" -# if command -v jq >/dev/null 2>&1; then -# JQ="$(command -v jq)" || fail "ERROR (Exit Code: ${?})" "${?}" -# export JQ -# else -# fail "ERROR (Exit Code: ${?})" "${?}" -# fi -# end_task "DONE (Found: ${JQ})" -#end_group -# -#start_group "Prepare the Docker Image Information" -# start_task "Resolving the DOCKER_REGISTRY variable" -# if [[ -z "${DOCKER_REGISTRY}" ]]; then -# DOCKER_REGISTRY="localhost:5000" -# fi -# end_task "DONE (Registry: ${DOCKER_REGISTRY})" -# + start_task "Resolving the GITHUB_WORKSPACE path" + # Ensure GITHUB_WORKSPACE is provided or default to the repository root + if [[ -z "${GITHUB_WORKSPACE}" || ! -d "${GITHUB_WORKSPACE}" ]]; then + GITHUB_WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../" && pwd)" + fi + end_task "DONE (Path: ${GITHUB_WORKSPACE})" + + start_task "Resolving the GITHUB_OUTPUT path" + # Ensure GITHUB_OUTPUT is provided or default to the repository root + if [[ -z "${GITHUB_OUTPUT}" ]]; then + GITHUB_OUTPUT="${TEMP_DIR}/workflow-output.txt" + fi + end_task "DONE (Path: ${GITHUB_OUTPUT})" + + start_task "Resolving the GITHUB_SHA hash" + if [[ -z "${GITHUB_SHA}" ]]; then + GITHUB_SHA="$(git rev-parse HEAD | tr -d '[:space:]')" || fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task "DONE (Commit: ${GITHUB_SHA})" + + start_task "Resolving the MANIFEST_PATH variable" + if [[ -z "${MANIFEST_PATH}" ]]; then + MANIFEST_PATH="${GITHUB_WORKSPACE}/.manifests/gradle" + fi + end_task "DONE (Path: ${MANIFEST_PATH})" + + start_task "Ensuring the MANIFEST_PATH location is present" + if [[ ! -d "${MANIFEST_PATH}" ]]; then + mkdir -p "${MANIFEST_PATH}" || fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task + + start_task "Resolving the SKOPEO_VERSION variable" + if [[ -z "${SKOPEO_VERSION}" ]]; then + SKOPEO_VERSION="v1.14.0" + fi + end_task "DONE (Version: ${SKOPEO_VERSION})" + + start_task "Resolving the SKOPEO_IMAGE_NAME variable" + if [[ -z "${SKOPEO_IMAGE_NAME}" ]]; then + SKOPEO_IMAGE_NAME="quay.io/skopeo/stable:${SKOPEO_VERSION}" + fi + end_task "DONE (Image Name: ${SKOPEO_IMAGE_NAME})" + + start_task "Checking for the DOCKER command" + if command -v docker >/dev/null 2>&1; then + DOCKER="$(command -v docker)" || fail "ERROR (Exit Code: ${?})" "${?}" + export DOCKER + else + fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task "DONE (Found: ${DOCKER})" + + start_task "Resolving the Docker Client Configuration" + SKOPEO_BIND_MOUNT="" + SKOPEO_CREDENTIAL_OPTS="" + DOCKER_CONFIG_DIR="${HOME}/.docker" + if [[ -d "${DOCKER_CONFIG_DIR}" ]]; then + SKOPEO_BIND_MOUNT="--volume ${DOCKER_CONFIG_DIR}:/tmp/docker" + SKOPEO_CREDENTIAL_OPTS="--authfile /tmp/docker/config.json" + fi + export SKOPEO_BIND_MOUNT SKOPEO_CREDENTIAL_OPTS + end_task "DONE" + + start_task "Checking for the SKOPEO command" + if command -v skopeo >/dev/null 2>&1; then + SKOPEO="$(command -v skopeo)" || fail "ERROR (Exit Code: ${?})" "${?}" + export SKOPEO + else + ${DOCKER} pull "${SKOPEO_IMAGE_NAME}" >/dev/null 2>&1 || fail "ERROR (Exit Code: ${?})" "${?}" + SKOPEO="${DOCKER} run ${SKOPEO_BIND_MOUNT} --rm --network host ${SKOPEO_IMAGE_NAME}" + export SKOPEO + fi + end_task "DONE (Found: ${SKOPEO})" + + start_task "Checking for the JQ command" + if command -v jq >/dev/null 2>&1; then + JQ="$(command -v jq)" || fail "ERROR (Exit Code: ${?})" "${?}" + export JQ + else + fail "ERROR (Exit Code: ${?})" "${?}" + fi + end_task "DONE (Found: ${JQ})" +end_group + +start_group "Prepare the Docker Image Information" + export DOCKER_REGISTRY DOCKER_TAG + + start_task "Resolving the DOCKER_REGISTRY variable" + if [[ -z "${DOCKER_REGISTRY}" ]]; then + DOCKER_REGISTRY="grpc.io" + fi + end_task "DONE (Registry: ${DOCKER_REGISTRY})" + # start_task "Resolving the DOCKER_TAG variable" # if [[ -z "${DOCKER_TAG}" ]]; then # DOCKER_TAG="$(echo "${GITHUB_SHA}" | tr -d '[:space:]' | cut -c1-8)" # fi # end_task "DONE (Tag: ${DOCKER_TAG})" -# -# start_task "Resolving the Fully Qualified Image Name" -# FQ_IMAGE_NAME="${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${DOCKER_TAG}" -# end_task "DONE (Image: ${FQ_IMAGE_NAME})" -#end_group -# -#start_group "Generate Docker Image Manifest (linux/amd64)" -# ${SKOPEO} --override-os linux --override-arch amd64 inspect ${SKOPEO_CREDENTIAL_OPTS} --tls-verify=false "docker://${FQ_IMAGE_NAME}" | tee "${TEMP_DIR}/linux-amd64.manifest.json" || fail "SKOPEO ERROR (Exit Code: ${?})" "${?}" -# ${JQ} -r '.Layers[]' "${TEMP_DIR}/linux-amd64.manifest.json" | tee "${TEMP_DIR}/linux-amd64.layers.json" >/dev/null 2>&1 || fail "JQ LAYER ERROR (Exit Code: ${?})" "${?}" -# ${JQ} -r 'del(.RepoTags) | del(.LayersData) | del(.Digest) | del(.Name)' "${TEMP_DIR}/linux-amd64.manifest.json" | tee "${TEMP_DIR}/linux-amd64.comparable.json" >/dev/null 2>&1 || fail "JQ COMP ERROR (Exit Code: ${?})" "${?}" -#end_group -# -#start_group "Generate Docker Image Manifest (linux/arm64)" -# ${SKOPEO} --override-os linux --override-arch arm64 inspect ${SKOPEO_CREDENTIAL_OPTS} --tls-verify=false "docker://${FQ_IMAGE_NAME}" | tee "${TEMP_DIR}/linux-arm64.manifest.json" || fail "SKOPEO ERROR (Exit Code: ${?})" "${?}" -# ${JQ} -r '.Layers[]' "${TEMP_DIR}/linux-arm64.manifest.json" | tee "${TEMP_DIR}/linux-arm64.layers.json" >/dev/null 2>&1 || fail "JQ LAYER ERROR (Exit Code: ${?})" "${?}" -# ${JQ} -r 'del(.RepoTags) | del(.LayersData) | del(.Digest) | del(.Name)' "${TEMP_DIR}/linux-arm64.manifest.json" | tee "${TEMP_DIR}/linux-arm64.comparable.json" >/dev/null 2>&1 || fail "JQ COMP ERROR (Exit Code: ${?})" "${?}" -#end_group -# -#start_group "Generating Final Release Manifests" -# -# start_task "Generating the manifest archive" -# MANIFEST_FILES=("linux-amd64.manifest.json" "linux-amd64.layers.json" "linux-amd64.comparable.json") -# MANIFEST_FILES+=("linux-arm64.manifest.json" "linux-arm64.layers.json" "linux-arm64.comparable.json") -# tar -czf "${TEMP_DIR}/manifest.tar.gz" -C "${TEMP_DIR}" "${MANIFEST_FILES[@]}" >/dev/null 2>&1 || fail "TAR ERROR (Exit Code: ${?})" "${?}" -# end_task -# -# start_task "Copying the manifest files" -# cp "${TEMP_DIR}/manifest.tar.gz" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" || fail "COPY ERROR (Exit Code: ${?})" "${?}" -# cp "${TEMP_DIR}"/*.json "${MANIFEST_PATH}/" || fail "COPY ERROR (Exit Code: ${?})" "${?}" -# end_task "DONE (Path: ${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz)" -# -# start_task "Setting Step Outputs" -# { -# printf "path=%s\n" "${MANIFEST_PATH}" -# printf "file=%s\n" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" -# printf "name=%s\n" "${GITHUB_SHA}.tar.gz" -# } >> "${GITHUB_OUTPUT}" -# end_task + + start_task "Resolving the Fully Qualified Image Name" + FQ_IMAGE_NAME="${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:0.3.0-SNAPSHOT" + end_task "DONE (Image: ${FQ_IMAGE_NAME})" +end_group + +start_group "Generate Docker Image Manifest (linux/amd64)" + ${SKOPEO} --override-os linux --override-arch amd64 inspect ${SKOPEO_CREDENTIAL_OPTS} --tls-verify=false "docker://${FQ_IMAGE_NAME}" | tee "${TEMP_DIR}/linux-amd64.manifest.json" || fail "SKOPEO ERROR (Exit Code: ${?})" "${?}" + ${JQ} -r '.Layers[]' "${TEMP_DIR}/linux-amd64.manifest.json" | tee "${TEMP_DIR}/linux-amd64.layers.json" >/dev/null 2>&1 || fail "JQ LAYER ERROR (Exit Code: ${?})" "${?}" + ${JQ} -r 'del(.RepoTags) | del(.LayersData) | del(.Digest) | del(.Name)' "${TEMP_DIR}/linux-amd64.manifest.json" | tee "${TEMP_DIR}/linux-amd64.comparable.json" >/dev/null 2>&1 || fail "JQ COMP ERROR (Exit Code: ${?})" "${?}" +end_group + +start_group "Generate Docker Image Manifest (linux/arm64)" + ${SKOPEO} --override-os linux --override-arch arm64 inspect ${SKOPEO_CREDENTIAL_OPTS} --tls-verify=false "docker://${FQ_IMAGE_NAME}" | tee "${TEMP_DIR}/linux-arm64.manifest.json" || fail "SKOPEO ERROR (Exit Code: ${?})" "${?}" + ${JQ} -r '.Layers[]' "${TEMP_DIR}/linux-arm64.manifest.json" | tee "${TEMP_DIR}/linux-arm64.layers.json" >/dev/null 2>&1 || fail "JQ LAYER ERROR (Exit Code: ${?})" "${?}" + ${JQ} -r 'del(.RepoTags) | del(.LayersData) | del(.Digest) | del(.Name)' "${TEMP_DIR}/linux-arm64.manifest.json" | tee "${TEMP_DIR}/linux-arm64.comparable.json" >/dev/null 2>&1 || fail "JQ COMP ERROR (Exit Code: ${?})" "${?}" +end_group + +start_group "Generating Final Release Manifests" + + start_task "Generating the manifest archive" + MANIFEST_FILES=("linux-amd64.manifest.json" "linux-amd64.layers.json" "linux-amd64.comparable.json") + MANIFEST_FILES+=("linux-arm64.manifest.json" "linux-arm64.layers.json" "linux-arm64.comparable.json") + tar -czf "${TEMP_DIR}/manifest.tar.gz" -C "${TEMP_DIR}" "${MANIFEST_FILES[@]}" >/dev/null 2>&1 || fail "TAR ERROR (Exit Code: ${?})" "${?}" + end_task + + start_task "Copying the manifest files" + cp "${TEMP_DIR}/manifest.tar.gz" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" || fail "COPY ERROR (Exit Code: ${?})" "${?}" + cp "${TEMP_DIR}"/*.json "${MANIFEST_PATH}/" || fail "COPY ERROR (Exit Code: ${?})" "${?}" + end_task "DONE (Path: ${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz)" + + start_task "Setting Step Outputs" + { + printf "path=%s\n" "${MANIFEST_PATH}" + printf "file=%s\n" "${MANIFEST_PATH}/${GITHUB_SHA}.tar.gz" + printf "name=%s\n" "${GITHUB_SHA}.tar.gz" + } >> "${GITHUB_OUTPUT}" + end_task end_group diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 7131aa7da..e1ec0e0bc 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -1,5 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 name: "ZXC: Verify Docker Build Determinism" +# Here, the ZXC prefix: +# Z - Ensures sort order such that this script appears at the bottom of the UI +# X - Indicates it's not for direct user consumption +# C - Indicates this is a 'workflow_call' based reusable workflow on: workflow_call: inputs: @@ -19,14 +23,6 @@ on: required: false default: "21.0.4" -# secrets: -# gradle-cache-username: -# description: "The username used to authenticate with the Gradle Build Cache Node." -# required: true -# gradle-cache-password: -# description: "The password used to authenticate with the Gradle Build Cache Node." -# required: true - defaults: run: shell: bash @@ -34,34 +30,439 @@ defaults: permissions: id-token: write contents: read + packages: write env: -# GRADLE_CACHE_USERNAME: ${{ secrets.gradle-cache-username }} -# GRADLE_CACHE_PASSWORD: ${{ secrets.gradle-cache-password }} DOCKER_MANIFEST_GENERATOR: .github/workflows/support/scripts/generate-docker-artifact-baseline.sh DOCKER_MANIFEST_PATH: ${{ github.workspace }}/.manifests/docker - DOCKER_REGISTRY: localhost:5000 - DOCKER_IMAGE_NAME: consensus-node - DOCKER_CONTEXT_PATH: hedera-node/infrastructure/docker/containers/production-next/consensus-node + DOCKER_REGISTRY: ghcr.io + DOCKER_IMAGE_NAME: hashgraph/hedera-block-node + DOCKER_CONTEXT_PATH: server/docker SKOPEO_VERSION: v1.14.0 + OWNER: hashgraph jobs: generate-baseline: name: Generate Baseline runs-on: block-node-linux-medium -# outputs: -# sha: ${{ steps.commit.outputs.sha }} -# sha-abbrev: ${{ steps.commit.outputs.sha-abbrev }} -# source-date: ${{ steps.commit.outputs.source-date }} -# path: ${{ steps.baseline.outputs.path }} -# file: ${{ steps.baseline.outputs.file }} -# name: ${{ steps.baseline.outputs.name }} + outputs: + sha: ${{ steps.commit.outputs.sha }} + sha-abbrev: ${{ steps.commit.outputs.sha-abbrev }} + source-date: ${{ steps.commit.outputs.source-date }} + path: ${{ steps.baseline.outputs.path }} + file: ${{ steps.baseline.outputs.file }} + name: ${{ steps.baseline.outputs.name }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - name: Standardize Git Line Endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Checkout Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref }} + + - name: Authenticate to Google Cloud + id: google-auth + uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7 + with: + workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" + service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" + + - name: Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 + + - name: Retrieve Commit Hash + id: commit + run: | + echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + echo "sha-abbrev=$(git rev-parse HEAD | tr -d '[:space:]' | cut -c1-8)" >> "${GITHUB_OUTPUT}" + echo "source-date=$(git log -1 --pretty=%ct)" >> "${GITHUB_OUTPUT}" + + - name: Baseline Existence Check + id: baseline + run: | + BASELINE_NAME="${{ steps.commit.outputs.sha }}.tar.gz" + BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines" + BASELINE_FILE="${BASELINE_PATH}/${BASELINE_NAME}" + BASELINE_EXISTS="false" + + if gsutil ls "${BASELINE_FILE}" >/dev/null 2>&1; then + BASELINE_EXISTS="false" + fi + + echo "exists=${BASELINE_EXISTS}" >> "${GITHUB_OUTPUT}" + echo "path=${BASELINE_PATH}" >> "${GITHUB_OUTPUT}" + echo "name=${BASELINE_NAME}" >> "${GITHUB_OUTPUT}" + echo "file=${BASELINE_FILE}" >> "${GITHUB_OUTPUT}" + + - name: Setup Java + uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + with: + distribution: ${{ inputs.java-distribution }} + java-version: ${{ inputs.java-version }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 # v4.2.1 + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + with: + cache-disabled: true + + - name: Install Skopeo and JQ + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends skopeo jq + + - name: Setup QEmu Support + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + + - name: Setup Docker Buildx Support + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + with: + version: v0.16.2 + driver-opts: network=host + + - name: Setup Local Docker Registry + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest + + - name: Show Docker Version + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: docker version + + - name: Show Docker Info + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: docker info + + - name: Build Gradle Artifacts + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: ./gradlew assemble --scan + + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + +# - name: Prepare for Docker Build +# if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} +# run: | +# mkdir -p "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data" +# +# echo "::group::Copying Application Artifacts" +# cp -Rvf "${{ github.workspace }}/server/build/libs" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data/" +# echo "::endgroup::" + +# - name: Write Artifact Version Descriptor +# run: | +# printf "VERSION=%s\nCOMMIT=%s\nDATE=%s" "$(./gradlew -q showVersion)" "$(git log -1 --format='%H' | cut -c1-8)" "$(date -u)" | tee "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/VERSION" + + - name: Extract version + id: extract_version + run: | + VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + + - name: Build Docker Image + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 + env: + SOURCE_DATE_EPOCH: ${{ steps.commit.outputs.source-date }} + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + with: + push: false + no-cache: true + platforms: linux/amd64,linux/arm64 + build-args: | + SOURCE_DATE_EPOCH=${{ steps.commit.outputs.source-date }} + VERSION=${{ env.VERSION }} + context: ./${{ env.DOCKER_CONTEXT_PATH }} + file: ./${{ env.DOCKER_CONTEXT_PATH }}/Dockerfile + tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ env.VERSION }} + build-contexts: | + distributions=./server/build/distributions + + - name: Generate Manifest + id: manifest + env: + MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }} + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} + + - name: Amend Manifest with Gradle Artifacts + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + working-directory: ${{ env.DOCKER_MANIFEST_PATH }} + run: | + STARTING_DIR=$(pwd) + echo "Starting directory: ${STARTING_DIR}" + EXTRACTED_FILE_NAME="${{ steps.commit.outputs.sha }}.tar" + echo "Before manifest operations ----------" + pwd + ls -la + gunzip "${{ steps.manifest.outputs.name }}" + echo "After gunzip ----------" + ls -la + tar -rvf "${EXTRACTED_FILE_NAME}" -C "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}" ../../server/build/ + gzip "${EXTRACTED_FILE_NAME}" + echo "After augmenting and gzipping ----------" + ls -la + +# - name: Clear Bucket +# run: | +# echo "::group::Clear Bucket Group" +# BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines" +# DOCKER_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/" +# gsutil ls "${DOCKER_PATH}" +# +# echo "Baseline Path: ${BASELINE_PATH}" +# gsutil ls "${BASELINE_PATH}/" +# echo "end of ls" +# +# gsutil rm "${BASELINE_PATH}/*" +# echo "end of rm" +# +# gsutil ls "${BASELINE_PATH}/" +# echo "end of second ls" +# echo "::endgroup::" + - name: Upload Baseline + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: gsutil cp "${{ steps.manifest.outputs.file }}" "${{ steps.baseline.outputs.file }}" + + verify-artifacts: + name: "Verify Artifacts (${{ join(matrix.os, ', ') }})" + runs-on: ${{ matrix.os }} + needs: + - generate-baseline + strategy: + fail-fast: false + matrix: + # Windows is not supported due to GitHub not supporting Docker Desktop/Podman Desktop and Docker CE on Windows + # not supporting BuildKit and the Buildx plugin. + # GitHub hosted MacOS and Ubuntu runners are temporarily disabled. + os: + #- ubuntu-22.04 + #- ubuntu-20.04 + #- macos-12 + #- macos-11 + - block-node-linux-medium steps: - name: Harden Runner uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 with: egress-policy: audit - - name: Hello World - run: echo "Testing Docker Build Determinism" + - name: Standardize Git Line Endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Checkout Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref }} + + - name: Setup Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: 3.9 + + - name: Install JQ (Linux) + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends jq + + - name: Install Skopeo (Linux) + if: ${{ runner.os == 'Linux' }} + run: | + source /etc/os-release + if [[ "${VERSION_ID}" != "20.04" ]]; then + sudo apt-get install --yes --no-install-recommends skopeo + fi + + - name: Authenticate to Google Cloud + id: google-auth + uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7 + with: + workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" + service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" + + - name: Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 + env: + CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} + + - name: Download Baseline + env: + CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} + run: | + echo "Starting directory: $(pwd)" + + echo "Creating build directory: ./server/build/distributions" + mkdir -p ./server/build/distributions + echo $? + echo "Here is the server dir:" + ls -la ./server/ + echo "Here is the server/build dir:" + ls -la ./server/build/ + echo "Here is the server/build/distributions/ dir:" + ls -la ./server/build/distributions/ + + echo "Making: ${{ env.DOCKER_MANIFEST_PATH }}" + mkdir -p "${{ env.DOCKER_MANIFEST_PATH }}" + cd "${{ env.DOCKER_MANIFEST_PATH }}" + + echo "Downloading: ${{ needs.generate-baseline.outputs.file }}" + gsutil cp "${{ needs.generate-baseline.outputs.file }}" . + echo "Extracting: ${{ needs.generate-baseline.outputs.name }}" + tar -xzf "${{ needs.generate-baseline.outputs.name }}" + pwd + ls -la + echo "listing sub dir ./server/build/distributions/" + ls -la ./server/build/distributions/ + + cd ./server/build/distributions + cp *.tar "${{ github.workspace }}/server/build/distributions/" + echo "Copied files to ${{ github.workspace }}/server/build/distributions/" + + echo "Here is the directory again:" + ls -la /home/runner/_work/hedera-block-node/hedera-block-node/server/build/distributions/ + + - name: Determine Home Directory + id: home + run: echo "directory=$(tr -d '[:space:]' < <(cd ~ && pwd))" >> "${GITHUB_OUTPUT}" + + - name: Setup QEmu Support + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + + - name: Setup Docker Buildx Support + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + with: + version: v0.16.2 + driver-opts: network=host + + - name: Setup Local Docker Registry + run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest + + - name: Show Docker Version + run: docker version + + - name: Show Docker Info + run: docker info + + + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract version + id: extract_version + run: | + VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) + echo "VERSION=${VERSION}" >> $GITHUB_ENV + + - name: Docker Image Debugging + run: | + echo "zero ---------------------" + pwd + ls -la + echo "${{ env.VERSION }}" + ls -la "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/" + echo "one ---------------------" + echo "server/" + ls -la ./server/ + echo "server/build/" + ls -la ./server/build/ + echo "server/build/distributions/" + ls -la ./server/build/distributions/ + + - name: Build Docker Image + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 + env: + SOURCE_DATE_EPOCH: ${{ needs.generate-baseline.outputs.source-date }} + with: + push: false + no-cache: true + platforms: linux/amd64,linux/arm64 + build-args: | + SOURCE_DATE_EPOCH=${{ needs.generate-baseline.outputs.source-date }} + VERSION=${{ env.VERSION }} + context: ./${{ env.DOCKER_CONTEXT_PATH }} + file: ./${{ env.DOCKER_CONTEXT_PATH }}/Dockerfile + tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ needs.generate-baseline.outputs.sha-abbrev }} + build-contexts: | + distributions=./server/build/distributions + + - name: Regenerate Manifest + id: regen-manifest + env: + MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }}/regenerated + run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} + + - name: Validate Layers (linux/amd64) + run: | + if ! diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" >/dev/null 2>&1; then + echo "::group::Debug Bucket Group" + BASELINE_NAME="${{ steps.commit.outputs.sha }}.tar.gz" + BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines" + BASELINE_FILE="${BASELINE_PATH}/${BASELINE_NAME}" + echo "Baseline Path: ${BASELINE_PATH}" + gsutil ls "${BASELINE_PATH}" + echo "end" + echo "::endgroup::" + + echo "::group::Layer Differences" + diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" + + echo "::endgroup::" + exit 1 + fi + + - name: Validate Layers (linux/arm64) + run: | + if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" >/dev/null 2>&1; then + echo "::group::Layer Differences" + diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" + echo "::endgroup::" + exit 1 + fi + + - name: Validate Full Manifest (linux/amd64) + run: | + if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" >/dev/null 2>&1; then + echo "::group::Layer Differences" + diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" + echo "::endgroup::" + exit 1 + fi + + - name: Validate Full Manifest (linux/arm64) + run: | + if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" >/dev/null 2>&1; then + echo "::group::Layer Differences" + diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" + echo "::endgroup::" + exit 1 + fi + + - name: Publish Manifests + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + if: ${{ steps.regen-manifest.conclusion == 'success' && failure() && !cancelled() }} + with: + name: Docker Manifests [${{ join(matrix.os, ', ') }}] + path: ${{ env.DOCKER_MANIFEST_PATH }}/** diff --git a/.github/workflows/zxc-verify-gradle-build-determinism.yaml b/.github/workflows/zxc-verify-gradle-build-determinism.yaml index 4cc5d9224..f274c88a1 100644 --- a/.github/workflows/zxc-verify-gradle-build-determinism.yaml +++ b/.github/workflows/zxc-verify-gradle-build-determinism.yaml @@ -35,8 +35,8 @@ defaults: shell: bash permissions: + id-token: write contents: read - packages: write env: GRADLE_MANIFEST_PATH: ${{ github.workspace }}/.manifests/gradle @@ -52,6 +52,7 @@ jobs: path: ${{ steps.baseline.outputs.path }} file: ${{ steps.baseline.outputs.file }} name: ${{ steps.baseline.outputs.name }} + steps: - name: Harden Runner uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 @@ -74,6 +75,16 @@ jobs: with: cache-disabled: true + - name: Authenticate to Google Cloud + id: google-auth + uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7 + with: + workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" + service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" + + - name: Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 + - name: Retrieve Commit Hash id: commit run: echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" @@ -97,14 +108,20 @@ jobs: - name: Build Artifacts id: gradle-build + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} run: ./gradlew assemble --scan - name: Generate Manifest id: manifest env: MANIFEST_PATH: ${{ env.GRADLE_MANIFEST_PATH }} + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} run: ${{ env.GRADLE_MANIFEST_GENERATOR }} + - name: Upload Baseline + if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} + run: gsutil cp "${{ steps.manifest.outputs.file }}" "${{ steps.baseline.outputs.file }}" + verify-artifacts: name: "Verify Artifacts (${{ join(matrix.os, ', ') }})" runs-on: ${{ matrix.os }} @@ -116,10 +133,7 @@ jobs: os: - ubuntu-22.04 - ubuntu-20.04 - - windows-2022 - - windows-2019 - block-node-linux-medium - - block-node-linux-large steps: - name: Harden Runner uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 @@ -152,30 +166,26 @@ jobs: with: cache-disabled: true -# - name: Setup CoreUtils (macOS) -# if: ${{ runner.os == 'macOS' }} -# run: brew install coreutils - -# - name: Authenticate to Google Cloud -# id: google-auth -# uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7 -# with: -# workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" -# service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" - -# - name: Setup Google Cloud SDK -# uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 -# env: -# CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} -# + - name: Authenticate to Google Cloud + id: google-auth + uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7 + with: + workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" + service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" + + - name: Setup Google Cloud SDK + uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2 + env: + CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} + - name: Download Baseline env: CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} run: | mkdir -p "${GRADLE_MANIFEST_PATH}" cd "${GRADLE_MANIFEST_PATH}" -# gsutil cp "${{ needs.generate-baseline.outputs.file }}" . -# tar -xzf "${{ needs.generate-baseline.outputs.name }}" + gsutil cp "${{ needs.generate-baseline.outputs.file }}" . + tar -xzf "${{ needs.generate-baseline.outputs.name }}" - name: Build Artifacts id: gradle-build @@ -191,18 +201,18 @@ jobs: working-directory: ${{ github.workspace }}/server/build/libs run: sha256sum -c "${GRADLE_MANIFEST_PATH}/applications.sha256" -# - name: Compare Application Manifests -# run: | -# if ! diff -u "${GRADLE_MANIFEST_PATH}/applications.sha256" "${{ steps.regen-manifest.outputs.applications }}" >/dev/null 2>&1; then -# echo "::group::Application Manifest Differences" -# diff -u "${GRADLE_MANIFEST_PATH}/applications.sha256" "${{ steps.regen-manifest.outputs.applications }}" -# echo "::endgroup::" -# exit 1 -# fi -# -# - name: Publish Manifests -# uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 -# if: ${{ steps.regen-manifest.conclusion == 'success' && failure() && !cancelled() }} -# with: -# name: Gradle Manifests [${{ join(matrix.os, ', ') }}] -# path: ${{ env.GRADLE_MANIFEST_PATH }}/** + - name: Compare Application Manifests + run: | + if ! diff -u "${GRADLE_MANIFEST_PATH}/applications.sha256" "${{ steps.regen-manifest.outputs.applications }}" >/dev/null 2>&1; then + echo "::group::Application Manifest Differences" + diff -u "${GRADLE_MANIFEST_PATH}/applications.sha256" "${{ steps.regen-manifest.outputs.applications }}" + echo "::endgroup::" + exit 1 + fi + + - name: Publish Manifests + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + if: ${{ steps.regen-manifest.conclusion == 'success' && failure() && !cancelled() }} + with: + name: Gradle Manifests [${{ join(matrix.os, ', ') }}] + path: ${{ env.GRADLE_MANIFEST_PATH }}/** diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 72b762f7c..f8ca9e409 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -3,22 +3,32 @@ # Define Global Build Arguments # ######################################################################################################################## -ARG UBUNTU_TAG="focal-20230605" +ARG UBUNTU_TAG="mantic-20240216" +ARG SOURCE_DATE_EPOCH="0" ######################################################################################################################## # # Setup Builder Image # ######################################################################################################################## -FROM ubuntu:${UBUNTU_TAG} AS openjdk-builder +FROM ubuntu:${UBUNTU_TAG} AS java-builder-interim +# Define Build Arguments +ARG SOURCE_DATE_EPOCH # Define Standard Environment Variables -ENV LC_ALL=C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 # Install basic OS utilities for building -RUN apt-get update && \ - apt-get install --yes tar gzip gnupg2 curl +RUN --mount=type=bind,source=./repro-sources-list.sh,target=/usr/local/bin/repro-sources-list.sh \ + repro-sources-list.sh && \ + apt-get update && \ + apt-get install --yes --no-install-recommends tar gzip curl ca-certificates && \ + apt-get autoclean --yes && \ + apt-get clean all --yes && \ + rm -rf /var/log/ && \ + rm -rf /var/cache/ ########################## #### Java Setup #### @@ -28,21 +38,21 @@ RUN set -eux; \ ARCH="$(dpkg --print-architecture)"; \ case "${ARCH}" in \ aarch64|arm64) \ - ESUM='e184dc29a6712c1f78754ab36fb48866583665fa345324f1a79e569c064f95e9'; \ - BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.1_12.tar.gz'; \ - ;; \ - amd64|i386:x86-64) \ - ESUM='1a6fa8abda4c5caed915cfbeeb176e7fbd12eb6b222f26e290ee45808b529aa1'; \ - BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_x64_linux_hotspot_21.0.1_12.tar.gz'; \ - ;; \ + ESUM='d768eecddd7a515711659e02caef8516b7b7177fa34880a56398fd9822593a79'; \ + BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.4_7.tar.gz'; \ + ;; \ + amd64|i386:x86-64) \ + ESUM='51fb4d03a4429c39d397d3a03a779077159317616550e4e71624c9843083e7b9'; \ + BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz'; \ + ;; \ ppc64el|powerpc:common64) \ - ESUM='9574828ef3d735a25404ced82e09bf20e1614f7d6403956002de9cfbfcb8638f'; \ - BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_ppc64le_linux_hotspot_21.0.1_12.tar.gz'; \ - ;; \ + ESUM='c208cd0fb90560644a90f928667d2f53bfe408c957a5e36206585ad874427761'; \ + BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_ppc64le_linux_hotspot_21.0.4_7.tar.gz'; \ + ;; \ *) \ - echo "Unsupported arch: ${ARCH}"; \ - exit 1; \ - ;; \ + echo "Unsupported arch: ${ARCH}"; \ + exit 1; \ + ;; \ esac; \ curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \ echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -; \ @@ -55,33 +65,48 @@ RUN set -eux; \ ; \ rm -f /tmp/openjdk.tar.gz /usr/local/java/lib/src.zip; +######################################## +#### Deterministic Build Hack #### +######################################## + +# === Workarounds below will not be needed when https://github.com/moby/buildkit/pull/4057 is merged === +# NOTE: PR #4057 has been merged but will not be available until the v0.13.x series of releases. +# Limit the timestamp upper bound to SOURCE_DATE_EPOCH. +# Workaround for https://github.com/moby/buildkit/issues/3180 +RUN find $( ls / | grep -E -v "^(dev|mnt|proc|sys)$" ) \ + -newermt "@${SOURCE_DATE_EPOCH}" -writable -xdev \ + | xargs touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference + +########################## + +FROM scratch AS java-builder +COPY --from=java-builder-interim / / + + ######################################################################################################################## # # Build Final Image # ######################################################################################################################## -FROM ubuntu:${UBUNTU_TAG} AS openjdk-base +FROM java-builder AS production-image + +# Define Build Arguments +ARG SOURCE_DATE_EPOCH # Define Standard Environment Variables +ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive -ENV JAVA_VERSION="jdk-21.0.1+12" -ENV JAVA_HOME="/usr/local/java/" - -# Fetch Validated Java Binaries -COPY --from=openjdk-builder /usr/local/java/ /usr/local/java/ -# Install Basic OS Requirements -RUN apt-get update && \ - apt-get install --yes --no-install-recommends tar gzip openssl curl && \ - apt-get autoremove --yes && \ - apt-get autoclean --yes && \ - apt-get clean all --yes && \ - rm -rf /var/lib/{apt,dpkg,cache,log}/ +ENV JAVA_VERSION="jdk-21.0.4+7" +ENV JAVA_HOME=/usr/local/java +ENV PATH=${JAVA_HOME}/bin:${PATH} +# Install Java +COPY --from=java-builder ${JAVA_HOME}/ ${JAVA_HOME}/ # Expose the port that the application will run on -EXPOSE 8080 +EXPOSE 8080/tcp # Define version ARG VERSION diff --git a/server/docker/docker-build.sh b/server/docker/docker-build.sh index c31c5bad4..d28680ea5 100755 --- a/server/docker/docker-build.sh +++ b/server/docker/docker-build.sh @@ -10,8 +10,12 @@ VERSION=$1 echo "Building image [block-node-server:${VERSION}]" echo +SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) # run docker build -docker buildx build --load -t "block-node-server:${VERSION}" --build-context distributions=../distributions --build-arg VERSION="${VERSION}" . || exit "${?}" +docker buildx build --load -t "block-node-server:${VERSION}" \ + --build-context distributions=../distributions \ + --build-arg VERSION="${VERSION}" \ + --build-arg SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH}" . || exit "${?}" echo echo "Image [block-node-server:${VERSION}] built successfully!" diff --git a/server/docker/repro-sources-list.sh b/server/docker/repro-sources-list.sh new file mode 100755 index 000000000..e822b7a9a --- /dev/null +++ b/server/docker/repro-sources-list.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# +# Copyright The repro-sources-list.sh Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ----------------------------------------------------------------------------- +# repro-sources-list.sh: +# configures /etc/apt/sources.list and similar files for installing packages from a snapshot. +# +# This script is expected to be executed inside Dockerfile. +# +# The following distributions are supported: +# - debian:11 (/etc/apt/sources.list) +# - debian:12 (/etc/apt/sources.list.d/debian.sources) +# - ubuntu:22.04 (/etc/apt/sources.list) +# - ubuntu:23.10 (/etc/apt/sources.list) +# - archlinux (/etc/pacman.d/mirrorlist) +# +# For the further information, see https://github.com/reproducible-containers/repro-sources-list.sh +# ----------------------------------------------------------------------------- + +set -eux -o pipefail + +. /etc/os-release + +keep_apt_cache() { + rm -f /etc/apt/apt.conf.d/docker-clean + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache +} + +case "${ID}" in +"debian") + # : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.debian.org/archive/}" + : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot-cloudflare.debian.org/archive/}" + : "${BACKPORTS:=}" + case "${VERSION_ID}" in + "10" | "11") + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list)}" + ;; + *) + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list.d/debian.sources)}" + rm -f /etc/apt/sources.list.d/debian.sources + ;; + esac + snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")" + # TODO: use the new format for Debian >= 12 + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME} main" >/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian-security/${snapshot} ${VERSION_CODENAME}-security main" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-updates main" >>/etc/apt/sources.list + if [ "${BACKPORTS}" = 1 ]; then echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list; fi + keep_apt_cache + ;; +"ubuntu") + : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.ubuntu.com/}" + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /etc/apt/sources.list)}" + snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")" + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} main restricted" >/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates main restricted" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} universe" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates universe" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME} multiverse" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-updates multiverse" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-backports main restricted universe multiverse" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security main restricted" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security universe" >>/etc/apt/sources.list + echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}ubuntu/${snapshot} ${VERSION_CODENAME}-security multiverse" >>/etc/apt/sources.list + keep_apt_cache + # http://snapshot.ubuntu.com is redirected to https, so we have to install ca-certificates + export DEBIAN_FRONTEND=noninteractive + apt-get -o Acquire::https::Verify-Peer=false update >&2 + apt-get -o Acquire::https::Verify-Peer=false install -y ca-certificates >&2 + ;; +"arch") + : "${SNAPSHOT_ARCHIVE_BASE:=http://archive.archlinux.org/}" + : "${SOURCE_DATE_EPOCH:=$(stat --format=%Y /var/log/pacman.log)}" + export SOURCE_DATE_EPOCH + # shellcheck disable=SC2016 + date -d "@${SOURCE_DATE_EPOCH}" "+Server = ${SNAPSHOT_ARCHIVE_BASE}repos/%Y/%m/%d/\$repo/os/\$arch" >/etc/pacman.d/mirrorlist + ;; +*) + echo >&2 "Unsupported distribution: ${ID}" + exit 1 + ;; +esac + +: "${WRITE_SOURCE_DATE_EPOCH:=/dev/null}" +echo "${SOURCE_DATE_EPOCH}" >"${WRITE_SOURCE_DATE_EPOCH}" +echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" From cb33f7453e419825018a80fd0dfbc7dc66a6fbe8 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 14:25:47 -0700 Subject: [PATCH 02/28] fix: trimmed out debugging info Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 115 ++++-------------- 1 file changed, 22 insertions(+), 93 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index e1ec0e0bc..e87a9ca3c 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -156,19 +156,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} -# - name: Prepare for Docker Build -# if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} -# run: | -# mkdir -p "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data" -# -# echo "::group::Copying Application Artifacts" -# cp -Rvf "${{ github.workspace }}/server/build/libs" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data/" -# echo "::endgroup::" - -# - name: Write Artifact Version Descriptor -# run: | -# printf "VERSION=%s\nCOMMIT=%s\nDATE=%s" "$(./gradlew -q showVersion)" "$(git log -1 --format='%H' | cut -c1-8)" "$(date -u)" | tee "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/VERSION" - - name: Extract version id: extract_version run: | @@ -204,37 +191,10 @@ jobs: if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} working-directory: ${{ env.DOCKER_MANIFEST_PATH }} run: | - STARTING_DIR=$(pwd) - echo "Starting directory: ${STARTING_DIR}" EXTRACTED_FILE_NAME="${{ steps.commit.outputs.sha }}.tar" - echo "Before manifest operations ----------" - pwd - ls -la gunzip "${{ steps.manifest.outputs.name }}" - echo "After gunzip ----------" - ls -la tar -rvf "${EXTRACTED_FILE_NAME}" -C "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}" ../../server/build/ gzip "${EXTRACTED_FILE_NAME}" - echo "After augmenting and gzipping ----------" - ls -la - -# - name: Clear Bucket -# run: | -# echo "::group::Clear Bucket Group" -# BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines" -# DOCKER_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/" -# gsutil ls "${DOCKER_PATH}" -# -# echo "Baseline Path: ${BASELINE_PATH}" -# gsutil ls "${BASELINE_PATH}/" -# echo "end of ls" -# -# gsutil rm "${BASELINE_PATH}/*" -# echo "end of rm" -# -# gsutil ls "${BASELINE_PATH}/" -# echo "end of second ls" -# echo "::endgroup::" - name: Upload Baseline if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} @@ -308,37 +268,16 @@ jobs: env: CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} run: | - echo "Starting directory: $(pwd)" - echo "Creating build directory: ./server/build/distributions" mkdir -p ./server/build/distributions - echo $? - echo "Here is the server dir:" - ls -la ./server/ - echo "Here is the server/build dir:" - ls -la ./server/build/ - echo "Here is the server/build/distributions/ dir:" - ls -la ./server/build/distributions/ - - echo "Making: ${{ env.DOCKER_MANIFEST_PATH }}" + mkdir -p "${{ env.DOCKER_MANIFEST_PATH }}" cd "${{ env.DOCKER_MANIFEST_PATH }}" - - echo "Downloading: ${{ needs.generate-baseline.outputs.file }}" gsutil cp "${{ needs.generate-baseline.outputs.file }}" . - echo "Extracting: ${{ needs.generate-baseline.outputs.name }}" tar -xzf "${{ needs.generate-baseline.outputs.name }}" - pwd - ls -la - echo "listing sub dir ./server/build/distributions/" - ls -la ./server/build/distributions/ - cd ./server/build/distributions cp *.tar "${{ github.workspace }}/server/build/distributions/" - echo "Copied files to ${{ github.workspace }}/server/build/distributions/" - - echo "Here is the directory again:" - ls -la /home/runner/_work/hedera-block-node/hedera-block-node/server/build/distributions/ + echo "Copied Block Node server tar file to ${{ github.workspace }}/server/build/distributions/" - name: Determine Home Directory id: home @@ -376,20 +315,20 @@ jobs: VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Docker Image Debugging - run: | - echo "zero ---------------------" - pwd - ls -la - echo "${{ env.VERSION }}" - ls -la "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/" - echo "one ---------------------" - echo "server/" - ls -la ./server/ - echo "server/build/" - ls -la ./server/build/ - echo "server/build/distributions/" - ls -la ./server/build/distributions/ +# - name: Docker Image Debugging +# run: | +# echo "zero ---------------------" +# pwd +# ls -la +# echo "${{ env.VERSION }}" +# ls -la "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/" +# echo "one ---------------------" +# echo "server/" +# ls -la ./server/ +# echo "server/build/" +# ls -la ./server/build/ +# echo "server/build/distributions/" +# ls -la ./server/build/distributions/ - name: Build Docker Image uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 @@ -417,45 +356,35 @@ jobs: - name: Validate Layers (linux/amd64) run: | if ! diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" >/dev/null 2>&1; then - echo "::group::Debug Bucket Group" - BASELINE_NAME="${{ steps.commit.outputs.sha }}.tar.gz" - BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines" - BASELINE_FILE="${BASELINE_PATH}/${BASELINE_NAME}" - echo "Baseline Path: ${BASELINE_PATH}" - gsutil ls "${BASELINE_PATH}" - echo "end" - echo "::endgroup::" - echo "::group::Layer Differences" diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" - echo "::endgroup::" exit 1 fi - name: Validate Layers (linux/arm64) run: | - if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" >/dev/null 2>&1; then + if ! diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" >/dev/null 2>&1; then echo "::group::Layer Differences" - diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" + diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" echo "::endgroup::" exit 1 fi - name: Validate Full Manifest (linux/amd64) run: | - if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" >/dev/null 2>&1; then + if ! diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" >/dev/null 2>&1; then echo "::group::Layer Differences" - diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" + diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" echo "::endgroup::" exit 1 fi - name: Validate Full Manifest (linux/arm64) run: | - if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" >/dev/null 2>&1; then + if ! diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" >/dev/null 2>&1; then echo "::group::Layer Differences" - diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" + diff -u "${{ env.DOCKER_MANIFEST_PATH }}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" echo "::endgroup::" exit 1 fi From e39cdbb49580ddfecc083082d88a41fa06aa0987 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 14:54:59 -0700 Subject: [PATCH 03/28] Removed GH Container Registry directives Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 44 +++++-------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index e87a9ca3c..9f0b47b90 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -149,12 +149,12 @@ jobs: if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} run: ./gradlew assemble --scan - - name: Login to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} +# - name: Login to GitHub Container Registry +# uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 +# with: +# registry: ${{ env.DOCKER_REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} - name: Extract version id: extract_version @@ -208,14 +208,9 @@ jobs: strategy: fail-fast: false matrix: - # Windows is not supported due to GitHub not supporting Docker Desktop/Podman Desktop and Docker CE on Windows - # not supporting BuildKit and the Buildx plugin. - # GitHub hosted MacOS and Ubuntu runners are temporarily disabled. os: #- ubuntu-22.04 #- ubuntu-20.04 - #- macos-12 - #- macos-11 - block-node-linux-medium steps: - name: Harden Runner @@ -302,12 +297,12 @@ jobs: run: docker info - - name: Login to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} +# - name: Login to GitHub Container Registry +# uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 +# with: +# registry: ${{ env.DOCKER_REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} - name: Extract version id: extract_version @@ -315,21 +310,6 @@ jobs: VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) echo "VERSION=${VERSION}" >> $GITHUB_ENV -# - name: Docker Image Debugging -# run: | -# echo "zero ---------------------" -# pwd -# ls -la -# echo "${{ env.VERSION }}" -# ls -la "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/" -# echo "one ---------------------" -# echo "server/" -# ls -la ./server/ -# echo "server/build/" -# ls -la ./server/build/ -# echo "server/build/distributions/" -# ls -la ./server/build/distributions/ - - name: Build Docker Image uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 env: From a88c4b61808671969f3b06e7876e99be0380d274 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 15:08:32 -0700 Subject: [PATCH 04/28] Enabling other supported linux versions Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 9f0b47b90..8949ab978 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -149,13 +149,6 @@ jobs: if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} run: ./gradlew assemble --scan -# - name: Login to GitHub Container Registry -# uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 -# with: -# registry: ${{ env.DOCKER_REGISTRY }} -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract version id: extract_version run: | @@ -209,8 +202,8 @@ jobs: fail-fast: false matrix: os: - #- ubuntu-22.04 - #- ubuntu-20.04 + - ubuntu-22.04 + - ubuntu-20.04 - block-node-linux-medium steps: - name: Harden Runner @@ -296,14 +289,6 @@ jobs: - name: Show Docker Info run: docker info - -# - name: Login to GitHub Container Registry -# uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 -# with: -# registry: ${{ env.DOCKER_REGISTRY }} -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract version id: extract_version run: | From 10338a1d0a422ca585c3cf48eff807046e3ae87e Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 15:31:06 -0700 Subject: [PATCH 05/28] Removed other linux version as they are not supported in hedera-services currently Signed-off-by: Matt Peterson --- .github/workflows/zxc-verify-docker-build-determinism.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 8949ab978..aa29ed421 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -202,8 +202,6 @@ jobs: fail-fast: false matrix: os: - - ubuntu-22.04 - - ubuntu-20.04 - block-node-linux-medium steps: - name: Harden Runner From d71e491a1b3422f52ed2587f13e66b18e5fec89b Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 15:56:26 -0700 Subject: [PATCH 06/28] fix: updated to use java 21.0.5+11 Signed-off-by: Matt Peterson --- .github/workflows/e2e-tests.yaml | 2 +- .github/workflows/pr-checks.yaml | 2 +- .github/workflows/release-automation.yaml | 4 ++-- .github/workflows/release-push-image.yaml | 6 +++--- .github/workflows/smoke-test.yaml | 2 +- .../zxc-verify-docker-build-determinism.yaml | 2 +- .../zxc-verify-gradle-build-determinism.yaml | 2 +- .../kotlin/com.hedera.block.conventions.gradle.kts | 2 +- server/docker/Dockerfile | 14 +++++--------- 9 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 2a45c909f..1127efc59 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -45,7 +45,7 @@ jobs: uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: "21.0.4" + java-version: "21.0.5" - name: Run Acceptance Tests id: acceptance-tests diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 656128898..35f2eee5b 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -46,7 +46,7 @@ jobs: uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: "21.0.4" + java-version: "21.0.5" - name: Cache Gradle packages uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 diff --git a/.github/workflows/release-automation.yaml b/.github/workflows/release-automation.yaml index 14bdef7e3..ea0389f7a 100644 --- a/.github/workflows/release-automation.yaml +++ b/.github/workflows/release-automation.yaml @@ -80,7 +80,7 @@ jobs: uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: "21.0.4" + java-version: "21.0.5" - name: Setup Gradle uses: gradle/actions/setup-gradle@d9c87d481d55275bb5441eef3fe0e46805f9ef70 # v3.5.0 @@ -181,7 +181,7 @@ jobs: uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: "21.0.4" + java-version: "21.0.5" - name: Setup Gradle uses: gradle/actions/setup-gradle@d9c87d481d55275bb5441eef3fe0e46805f9ef70 # v3.5.0 diff --git a/.github/workflows/release-push-image.yaml b/.github/workflows/release-push-image.yaml index c0823be86..e5aa7fa86 100644 --- a/.github/workflows/release-push-image.yaml +++ b/.github/workflows/release-push-image.yaml @@ -36,7 +36,7 @@ jobs: with: ref: ${{ github.event.inputs.ref || '' }} java-distribution: ${{ inputs.java-distribution || 'temurin' }} - java-version: ${{ inputs.java-version || '21.0.4' }} + java-version: ${{ inputs.java-version || '21.0.5' }} check-docker: name: Docker @@ -44,7 +44,7 @@ jobs: with: ref: ${{ github.event.inputs.ref || '' }} java-distribution: ${{ inputs.java-distribution || 'temurin' }} - java-version: ${{ inputs.java-version || '21.0.4' }} + java-version: ${{ inputs.java-version || '21.0.5' }} publish: runs-on: block-node-linux-medium @@ -62,7 +62,7 @@ jobs: uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: "21.0.4" + java-version: "21.0.5" - name: Build run: ./gradlew clean build diff --git a/.github/workflows/smoke-test.yaml b/.github/workflows/smoke-test.yaml index 4bc46a5ce..63f9f52c7 100644 --- a/.github/workflows/smoke-test.yaml +++ b/.github/workflows/smoke-test.yaml @@ -46,7 +46,7 @@ jobs: uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 with: distribution: "temurin" - java-version: "21.0.4" + java-version: "21.0.5" - name: Install grpcurl run: | diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index aa29ed421..e31af3304 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -21,7 +21,7 @@ on: description: "Java JDK Version:" type: string required: false - default: "21.0.4" + default: "21.0.5" defaults: run: diff --git a/.github/workflows/zxc-verify-gradle-build-determinism.yaml b/.github/workflows/zxc-verify-gradle-build-determinism.yaml index f274c88a1..2f372acc6 100644 --- a/.github/workflows/zxc-verify-gradle-build-determinism.yaml +++ b/.github/workflows/zxc-verify-gradle-build-determinism.yaml @@ -21,7 +21,7 @@ on: description: "Java JDK Version:" type: string required: false - default: "21.0.4" + default: "21.0.5" # workflow_dispatch: # inputs: diff --git a/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts b/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts index 0545f133c..ed7030373 100644 --- a/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.hedera.block.conventions.gradle.kts @@ -17,7 +17,7 @@ plugins { group = "com.hedera.block" val javaVersionMajor = JavaVersion.VERSION_21 -val javaVersionPatch = "0.4" +val javaVersionPatch = "0.5" val currentJavaVersionMajor = JavaVersion.current() val currentJavaVersion = providers.systemProperty("java.version").get() diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index f8ca9e409..5fe0df083 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -38,16 +38,12 @@ RUN set -eux; \ ARCH="$(dpkg --print-architecture)"; \ case "${ARCH}" in \ aarch64|arm64) \ - ESUM='d768eecddd7a515711659e02caef8516b7b7177fa34880a56398fd9822593a79'; \ - BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.4_7.tar.gz'; \ + ESUM='6482639ed9fd22aa2e704cc366848b1b3e1586d2bf1213869c43e80bca58fe5c'; \ + BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.5%2B11/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.5_11.tar.gz' \ ;; \ amd64|i386:x86-64) \ - ESUM='51fb4d03a4429c39d397d3a03a779077159317616550e4e71624c9843083e7b9'; \ - BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_x64_linux_hotspot_21.0.4_7.tar.gz'; \ - ;; \ - ppc64el|powerpc:common64) \ - ESUM='c208cd0fb90560644a90f928667d2f53bfe408c957a5e36206585ad874427761'; \ - BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.4%2B7/OpenJDK21U-jdk_ppc64le_linux_hotspot_21.0.4_7.tar.gz'; \ + ESUM=''; \ + BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.5%2B11/OpenJDK21U-jdk_x64_linux_hotspot_21.0.5_11.tar.gz' \ ;; \ *) \ echo "Unsupported arch: ${ARCH}"; \ @@ -98,7 +94,7 @@ ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive -ENV JAVA_VERSION="jdk-21.0.4+7" +ENV JAVA_VERSION="jdk-21.0.5+11" ENV JAVA_HOME=/usr/local/java ENV PATH=${JAVA_HOME}/bin:${PATH} From 2b4028a8cce672dd6adaf3e8251bc03d2cf07a4e Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Mon, 6 Jan 2025 16:04:14 -0700 Subject: [PATCH 07/28] Fixed missing ESUM Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 5fe0df083..067eb9610 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -42,7 +42,7 @@ RUN set -eux; \ BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.5%2B11/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.5_11.tar.gz' \ ;; \ amd64|i386:x86-64) \ - ESUM=''; \ + ESUM='3c654d98404c073b8a7e66bffb27f4ae3e7ede47d13284c132d40a83144bfd8c'; \ BINARY_URL='https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.5%2B11/OpenJDK21U-jdk_x64_linux_hotspot_21.0.5_11.tar.gz' \ ;; \ *) \ From bd2283dd55b29343825cd92177bb35da69776788 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Tue, 7 Jan 2025 13:07:56 -0700 Subject: [PATCH 08/28] Use the dynamic tag Signed-off-by: Matt Peterson --- .../generate-docker-artifact-baseline.sh | 9 +------ .../zxc-verify-docker-build-determinism.yaml | 27 ++++++++++++++++--- server/docker/repro-sources-list.sh | 2 -- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh b/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh index 4303a5f2a..853d22031 100755 --- a/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh +++ b/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh @@ -3,7 +3,6 @@ set -o pipefail set +e readonly DOCKER_IMAGE_NAME="hashgraph/hedera-block-node" -#readonly DOCKER_REGISTRY="ghcr.io" GROUP_ACTIVE="false" @@ -157,14 +156,8 @@ start_group "Prepare the Docker Image Information" fi end_task "DONE (Registry: ${DOCKER_REGISTRY})" -# start_task "Resolving the DOCKER_TAG variable" -# if [[ -z "${DOCKER_TAG}" ]]; then -# DOCKER_TAG="$(echo "${GITHUB_SHA}" | tr -d '[:space:]' | cut -c1-8)" -# fi -# end_task "DONE (Tag: ${DOCKER_TAG})" - start_task "Resolving the Fully Qualified Image Name" - FQ_IMAGE_NAME="${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:0.3.0-SNAPSHOT" + FQ_IMAGE_NAME="${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${GITHUB_SHA}" end_task "DONE (Image: ${FQ_IMAGE_NAME})" end_group diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index e31af3304..f6bcadc68 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -149,6 +149,13 @@ jobs: if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} run: ./gradlew assemble --scan + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract version id: extract_version run: | @@ -161,7 +168,7 @@ jobs: SOURCE_DATE_EPOCH: ${{ steps.commit.outputs.source-date }} if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} with: - push: false + push: true no-cache: true platforms: linux/amd64,linux/arm64 build-args: | @@ -169,7 +176,7 @@ jobs: VERSION=${{ env.VERSION }} context: ./${{ env.DOCKER_CONTEXT_PATH }} file: ./${{ env.DOCKER_CONTEXT_PATH }}/Dockerfile - tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ env.VERSION }} + tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.commit.outputs.sha-abbrev }} build-contexts: | distributions=./server/build/distributions @@ -178,7 +185,7 @@ jobs: env: MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }} if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} - run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} + run: GITHUB_SHA="${{ steps.commit.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} - name: Amend Manifest with Gradle Artifacts if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} @@ -287,18 +294,30 @@ jobs: - name: Show Docker Info run: docker info + - name: Login to GitHub Container Registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract version id: extract_version run: | VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Docker Debug + id: docker_debug + run: | + echo "sha-abbrev: ${{ needs.generate-baseline.outputs.sha-abbrev }}" + - name: Build Docker Image uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 env: SOURCE_DATE_EPOCH: ${{ needs.generate-baseline.outputs.source-date }} with: - push: false + push: true no-cache: true platforms: linux/amd64,linux/arm64 build-args: | diff --git a/server/docker/repro-sources-list.sh b/server/docker/repro-sources-list.sh index e822b7a9a..1bd0b01ff 100755 --- a/server/docker/repro-sources-list.sh +++ b/server/docker/repro-sources-list.sh @@ -41,7 +41,6 @@ keep_apt_cache() { case "${ID}" in "debian") - # : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot.debian.org/archive/}" : "${SNAPSHOT_ARCHIVE_BASE:=http://snapshot-cloudflare.debian.org/archive/}" : "${BACKPORTS:=}" case "${VERSION_ID}" in @@ -54,7 +53,6 @@ case "${ID}" in ;; esac snapshot="$(printf "%(%Y%m%dT%H%M%SZ)T\n" "${SOURCE_DATE_EPOCH}")" - # TODO: use the new format for Debian >= 12 echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME} main" >/etc/apt/sources.list echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian-security/${snapshot} ${VERSION_CODENAME}-security main" >>/etc/apt/sources.list echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-updates main" >>/etc/apt/sources.list From 2a40ac7259e18c8b8e9233bd50ef14f91f3129f4 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 04:36:58 -0700 Subject: [PATCH 09/28] fix: fixed the sha abbrev image ref Signed-off-by: Matt Peterson --- .../support/scripts/generate-docker-artifact-baseline.sh | 9 ++++++++- .../workflows/zxc-verify-docker-build-determinism.yaml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh b/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh index 853d22031..18924bad0 100755 --- a/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh +++ b/.github/workflows/support/scripts/generate-docker-artifact-baseline.sh @@ -156,8 +156,15 @@ start_group "Prepare the Docker Image Information" fi end_task "DONE (Registry: ${DOCKER_REGISTRY})" + start_task "Resolving the DOCKER_TAG variable" + if [[ -z "${DOCKER_TAG}" ]]; then + DOCKER_TAG="$(echo "${GITHUB_SHA}" | tr -d '[:space:]' | cut -c1-8)" + fi + end_task "DONE (Tag: ${DOCKER_TAG})" + + start_task "Resolving the Fully Qualified Image Name" - FQ_IMAGE_NAME="${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${GITHUB_SHA}" + FQ_IMAGE_NAME="${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${DOCKER_TAG}" end_task "DONE (Image: ${FQ_IMAGE_NAME})" end_group diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index f6bcadc68..4c1aec114 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -185,7 +185,7 @@ jobs: env: MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }} if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} - run: GITHUB_SHA="${{ steps.commit.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} + run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} - name: Amend Manifest with Gradle Artifacts if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} From 732c077ce024a8612f4a0508fab237c7f6ee16ea Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 04:59:11 -0700 Subject: [PATCH 10/28] fix: changed docker version parameter to use sha Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 4c1aec114..341f653fa 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -301,11 +301,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract version - id: extract_version - run: | - VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) - echo "VERSION=${VERSION}" >> $GITHUB_ENV +# - name: Extract version +# id: extract_version +# run: | +# VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) +# echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Docker Debug id: docker_debug @@ -322,7 +322,7 @@ jobs: platforms: linux/amd64,linux/arm64 build-args: | SOURCE_DATE_EPOCH=${{ needs.generate-baseline.outputs.source-date }} - VERSION=${{ env.VERSION }} + VERSION=${{ needs.generate-baseline.outputs.sha-abbrev }} context: ./${{ env.DOCKER_CONTEXT_PATH }} file: ./${{ env.DOCKER_CONTEXT_PATH }}/Dockerfile tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ needs.generate-baseline.outputs.sha-abbrev }} From 80123a79e196b412869d9d19b3b56c734d289dba Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 05:17:39 -0700 Subject: [PATCH 11/28] fix: restored VERSION and changed SHA param Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 341f653fa..35a21dbf6 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -185,7 +185,7 @@ jobs: env: MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }} if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} - run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} + run: GITHUB_SHA="${{ steps.commit.outputs.sha }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} - name: Amend Manifest with Gradle Artifacts if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} @@ -301,11 +301,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} -# - name: Extract version -# id: extract_version -# run: | -# VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) -# echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Extract version + id: extract_version + run: | + VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) + echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Docker Debug id: docker_debug @@ -322,7 +322,7 @@ jobs: platforms: linux/amd64,linux/arm64 build-args: | SOURCE_DATE_EPOCH=${{ needs.generate-baseline.outputs.source-date }} - VERSION=${{ needs.generate-baseline.outputs.sha-abbrev }} + VERSION=${{ env.VERSION }} context: ./${{ env.DOCKER_CONTEXT_PATH }} file: ./${{ env.DOCKER_CONTEXT_PATH }}/Dockerfile tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ needs.generate-baseline.outputs.sha-abbrev }} @@ -333,7 +333,7 @@ jobs: id: regen-manifest env: MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }}/regenerated - run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} + run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} - name: Validate Layers (linux/amd64) run: | From 5ee0dfe2851dad1884ae8fa63bb039705c8dff43 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 06:39:44 -0700 Subject: [PATCH 12/28] moved useradd up in the layers Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 067eb9610..b28b7d44b 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -73,6 +73,16 @@ RUN find $( ls / | grep -E -v "^(dev|mnt|proc|sys)$" ) \ -newermt "@${SOURCE_DATE_EPOCH}" -writable -xdev \ | xargs touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference +# Create a non-root user and group +ARG UNAME=hedera +ARG UID=2000 +ARG GID=2000 + +# Configure the standard user account +RUN groupadd --gid ${GID} ${UNAME} && \ + useradd --no-user-group --create-home --uid ${UID} --gid ${GID} --shell /bin/bash ${UNAME} + + ########################## FROM scratch AS java-builder @@ -107,16 +117,7 @@ EXPOSE 8080/tcp # Define version ARG VERSION -# Create a non-root user and group -ARG UNAME=hedera -ARG UID=2000 -ARG GID=2000 - -# Configure the standard user account -RUN groupadd --gid ${GID} ${UNAME} && \ - useradd --no-user-group --create-home --uid ${UID} --gid ${GID} --shell /bin/bash ${UNAME} - -USER $UNAME +USER hedera # Set the working directory inside the container WORKDIR /app From da26594ab86bc48af51995ace198437b42c2bc34 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 07:00:12 -0700 Subject: [PATCH 13/28] Removed env vars during useradd Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index b28b7d44b..8c315bbe9 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -73,15 +73,8 @@ RUN find $( ls / | grep -E -v "^(dev|mnt|proc|sys)$" ) \ -newermt "@${SOURCE_DATE_EPOCH}" -writable -xdev \ | xargs touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference -# Create a non-root user and group -ARG UNAME=hedera -ARG UID=2000 -ARG GID=2000 - -# Configure the standard user account -RUN groupadd --gid ${GID} ${UNAME} && \ - useradd --no-user-group --create-home --uid ${UID} --gid ${GID} --shell /bin/bash ${UNAME} - +RUN groupadd --gid 2000 hedera && \ + useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera ########################## From ae7b9cdea577bc48d27a80c5909a2d03826beccc Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 07:58:52 -0700 Subject: [PATCH 14/28] moved the useradd above the workaround Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 8c315bbe9..222fc36a3 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -65,6 +65,9 @@ RUN set -eux; \ #### Deterministic Build Hack #### ######################################## +RUN groupadd --gid 2000 hedera && \ + useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera \ + # === Workarounds below will not be needed when https://github.com/moby/buildkit/pull/4057 is merged === # NOTE: PR #4057 has been merged but will not be available until the v0.13.x series of releases. # Limit the timestamp upper bound to SOURCE_DATE_EPOCH. @@ -73,9 +76,6 @@ RUN find $( ls / | grep -E -v "^(dev|mnt|proc|sys)$" ) \ -newermt "@${SOURCE_DATE_EPOCH}" -writable -xdev \ | xargs touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference -RUN groupadd --gid 2000 hedera && \ - useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera - ########################## FROM scratch AS java-builder From 9db6525e704e5120021ed2da951cf83635bbd675 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 08:08:05 -0700 Subject: [PATCH 15/28] removed backslash char Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 222fc36a3..71539a5fe 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -66,7 +66,7 @@ RUN set -eux; \ ######################################## RUN groupadd --gid 2000 hedera && \ - useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera \ + useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera # === Workarounds below will not be needed when https://github.com/moby/buildkit/pull/4057 is merged === # NOTE: PR #4057 has been merged but will not be available until the v0.13.x series of releases. From 5e385220d46d1809c80ad1686f11d8a47cf68cc5 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 08:41:52 -0700 Subject: [PATCH 16/28] moved some operations up Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 71539a5fe..cbd288ec1 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -68,6 +68,25 @@ RUN set -eux; \ RUN groupadd --gid 2000 hedera && \ useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera +# Define version +ARG VERSION + +#USER hedera + +# Set the working directory inside the container +WORKDIR /app + +# Copy Distribution TAR file +COPY --from=distributions server-${VERSION}.tar . + +# Extract the TAR file +RUN tar -xvf server-${VERSION}.tar + +# Copy the logging properties file +COPY logging.properties logging.properties + +WORKDIR / + # === Workarounds below will not be needed when https://github.com/moby/buildkit/pull/4057 is merged === # NOTE: PR #4057 has been merged but will not be available until the v0.13.x series of releases. # Limit the timestamp upper bound to SOURCE_DATE_EPOCH. @@ -107,23 +126,9 @@ COPY --from=java-builder ${JAVA_HOME}/ ${JAVA_HOME}/ # Expose the port that the application will run on EXPOSE 8080/tcp -# Define version -ARG VERSION - USER hedera - -# Set the working directory inside the container WORKDIR /app -# Copy Distribution TAR file -COPY --from=distributions server-${VERSION}.tar . - -# Extract the TAR file -RUN tar -xvf server-${VERSION}.tar - -# Copy the logging properties file -COPY logging.properties logging.properties - # HEALTHCHECK for liveness and readiness HEALTHCHECK --interval=30s --timeout=10s --start-period=3s --retries=3 \ CMD curl -f http://localhost:8080/healthz/livez || exit 1 && \ From 7ed4867c1ca1f6b67b11686fec5fc6816b5ffa9f Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 08:54:24 -0700 Subject: [PATCH 17/28] fixed /app ownership Signed-off-by: Matt Peterson --- server/docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index cbd288ec1..b06f0e000 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -71,8 +71,6 @@ RUN groupadd --gid 2000 hedera && \ # Define version ARG VERSION -#USER hedera - # Set the working directory inside the container WORKDIR /app @@ -87,6 +85,9 @@ COPY logging.properties logging.properties WORKDIR / +# Ensure proper file permissions +RUN chown -R 2000:2000 /app + # === Workarounds below will not be needed when https://github.com/moby/buildkit/pull/4057 is merged === # NOTE: PR #4057 has been merged but will not be available until the v0.13.x series of releases. # Limit the timestamp upper bound to SOURCE_DATE_EPOCH. From a3fdbcbb3725fc12f114c66387c5a94190af8ab7 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 09:24:49 -0700 Subject: [PATCH 18/28] fix: added buildx caching support Signed-off-by: Matt Peterson --- .github/workflows/zxc-verify-docker-build-determinism.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 35a21dbf6..fbb79c092 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -132,6 +132,9 @@ jobs: with: version: v0.16.2 driver-opts: network=host + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"] - name: Setup Local Docker Registry if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} @@ -284,6 +287,9 @@ jobs: with: version: v0.16.2 driver-opts: network=host + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"] - name: Setup Local Docker Registry run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest From cfcf03b0ba3fdbfd18cad01097e09202462d750b Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 09:26:26 -0700 Subject: [PATCH 19/28] added caching support to the main publish buildx Signed-off-by: Matt Peterson --- .github/workflows/release-push-image.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release-push-image.yaml b/.github/workflows/release-push-image.yaml index e5aa7fa86..a150634c9 100644 --- a/.github/workflows/release-push-image.yaml +++ b/.github/workflows/release-push-image.yaml @@ -81,6 +81,9 @@ jobs: uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0 with: driver-opts: network=host + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"] - name: Extract version id: extract_version From 164bcf5df4e6f8b91c41353641b245a247380e7f Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 09:40:44 -0700 Subject: [PATCH 20/28] removed debugging code Signed-off-by: Matt Peterson --- .github/workflows/zxc-verify-docker-build-determinism.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index fbb79c092..876393a53 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -313,11 +313,6 @@ jobs: VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Docker Debug - id: docker_debug - run: | - echo "sha-abbrev: ${{ needs.generate-baseline.outputs.sha-abbrev }}" - - name: Build Docker Image uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0 env: From 96adbd160c4a5a4fa4597312683b968097e259c5 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Wed, 8 Jan 2025 16:34:59 -0700 Subject: [PATCH 21/28] Added workflow_dispatch for both zxc files Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 18 +++++++++++++++ .../zxc-verify-gradle-build-determinism.yaml | 23 ++++++++++++++----- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 876393a53..4f1c2c130 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -23,6 +23,24 @@ on: required: false default: "21.0.5" + workflow_dispatch: + inputs: + ref: + description: "The branch, tag, or commit to checkout:" + type: string + required: false + default: "" + java-distribution: + description: "Java JDK Distribution:" + type: string + required: false + default: "temurin" + java-version: + description: "Java JDK Version:" + type: string + required: false + default: "21.0.5" + defaults: run: shell: bash diff --git a/.github/workflows/zxc-verify-gradle-build-determinism.yaml b/.github/workflows/zxc-verify-gradle-build-determinism.yaml index 2f372acc6..ecc868792 100644 --- a/.github/workflows/zxc-verify-gradle-build-determinism.yaml +++ b/.github/workflows/zxc-verify-gradle-build-determinism.yaml @@ -23,12 +23,23 @@ on: required: false default: "21.0.5" -# workflow_dispatch: -# inputs: -# version: -# description: 'Release tag:' -# type: string -# required: false + workflow_dispatch: + inputs: + ref: + description: "The branch, tag, or commit to checkout:" + type: string + required: false + default: "" + java-distribution: + description: "Java JDK Distribution:" + type: string + required: false + default: "temurin" + java-version: + description: "Java JDK Version:" + type: string + required: false + default: "21.0.5" defaults: run: From 4bc480d7ed2532e33e1364784b633187bc7a9667 Mon Sep 17 00:00:00 2001 From: Jendrik Johannes Date: Fri, 10 Jan 2025 08:10:51 +0100 Subject: [PATCH 22/28] ci: fix new extract version steps Signed-off-by: Jendrik Johannes --- .github/workflows/zxc-verify-docker-build-determinism.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index 4f1c2c130..f99b44103 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -180,7 +180,7 @@ jobs: - name: Extract version id: extract_version run: | - VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) + VERSION=$(cat version.txt) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Build Docker Image @@ -328,7 +328,7 @@ jobs: - name: Extract version id: extract_version run: | - VERSION=$(grep 'version=' gradle.properties | cut -d '=' -f2) + VERSION=$(cat version.txt) echo "VERSION=${VERSION}" >> $GITHUB_ENV - name: Build Docker Image From 8b03b5ecb664e03e14b5effb856d451c535cb45e Mon Sep 17 00:00:00 2001 From: Jendrik Johannes Date: Fri, 10 Jan 2025 08:34:17 +0100 Subject: [PATCH 23/28] ci: checkout fix for spotless in 'release-push-image.yaml' Signed-off-by: Jendrik Johannes --- .github/workflows/release-push-image.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release-push-image.yaml b/.github/workflows/release-push-image.yaml index de1f00a85..43f67dc53 100644 --- a/.github/workflows/release-push-image.yaml +++ b/.github/workflows/release-push-image.yaml @@ -57,6 +57,8 @@ jobs: - name: Checkout repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 - name: Install JDK uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 From ee0d9ef21a0e9aa98752d9a709a1104915f46bd9 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Fri, 10 Jan 2025 12:56:20 -0700 Subject: [PATCH 24/28] fix: pushing to a local registry Signed-off-by: Matt Peterson --- .../zxc-verify-docker-build-determinism.yaml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/zxc-verify-docker-build-determinism.yaml b/.github/workflows/zxc-verify-docker-build-determinism.yaml index f99b44103..5c08312aa 100644 --- a/.github/workflows/zxc-verify-docker-build-determinism.yaml +++ b/.github/workflows/zxc-verify-docker-build-determinism.yaml @@ -53,7 +53,7 @@ permissions: env: DOCKER_MANIFEST_GENERATOR: .github/workflows/support/scripts/generate-docker-artifact-baseline.sh DOCKER_MANIFEST_PATH: ${{ github.workspace }}/.manifests/docker - DOCKER_REGISTRY: ghcr.io + DOCKER_REGISTRY: localhost:5000 DOCKER_IMAGE_NAME: hashgraph/hedera-block-node DOCKER_CONTEXT_PATH: server/docker SKOPEO_VERSION: v1.14.0 @@ -170,13 +170,6 @@ jobs: if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} run: ./gradlew assemble --scan - - name: Login to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract version id: extract_version run: | @@ -318,13 +311,6 @@ jobs: - name: Show Docker Info run: docker info - - name: Login to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 - with: - registry: ${{ env.DOCKER_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract version id: extract_version run: | From a311d635498f95d9880f76ec327c5aecfd48bd54 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Fri, 10 Jan 2025 13:09:43 -0700 Subject: [PATCH 25/28] fix: added a workflow dependency Signed-off-by: Matt Peterson --- .github/workflows/release-push-image.yaml | 1 + server/docker/Dockerfile | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-push-image.yaml b/.github/workflows/release-push-image.yaml index 43f67dc53..464e7a30c 100644 --- a/.github/workflows/release-push-image.yaml +++ b/.github/workflows/release-push-image.yaml @@ -47,6 +47,7 @@ jobs: java-version: ${{ inputs.java-version || '21.0.5' }} publish: + needs: [check-gradle, check-docker] runs-on: block-node-linux-medium steps: diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 9452e697a..7adb26ace 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -61,9 +61,6 @@ RUN set -eux; \ ; \ rm -f /tmp/openjdk.tar.gz /usr/local/java/lib/src.zip; -######################################## -#### Deterministic Build Hack #### -######################################## RUN groupadd --gid 2000 hedera && \ useradd --no-user-group --create-home --uid 2000 --gid 2000 --shell /bin/bash hedera @@ -91,6 +88,10 @@ WORKDIR / # Ensure proper file permissions RUN chown -R 2000:2000 /app +######################################## +#### Deterministic Build Hack #### +######################################## + # === Workarounds below will not be needed when https://github.com/moby/buildkit/pull/4057 is merged === # NOTE: PR #4057 has been merged but will not be available until the v0.13.x series of releases. # Limit the timestamp upper bound to SOURCE_DATE_EPOCH. From 7fd7756017c5a28e6c309a989d4c70e0c29feca7 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Fri, 10 Jan 2025 13:17:53 -0700 Subject: [PATCH 26/28] fix: added the determinism workflow checks for PRs Signed-off-by: Matt Peterson --- .github/workflows/pr-checks.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 8bf0c9f8d..ada34377a 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -20,6 +20,22 @@ permissions: contents: read jobs: + check-gradle: + name: Gradle + uses: ./.github/workflows/zxc-verify-gradle-build-determinism.yaml + with: + ref: ${{ github.event.inputs.ref || '' }} + java-distribution: ${{ inputs.java-distribution || 'temurin' }} + java-version: ${{ inputs.java-version || '21.0.5' }} + + check-docker: + name: Docker + uses: ./.github/workflows/zxc-verify-docker-build-determinism.yaml + with: + ref: ${{ github.event.inputs.ref || '' }} + java-distribution: ${{ inputs.java-distribution || 'temurin' }} + java-version: ${{ inputs.java-version || '21.0.5' }} + compile: name: "Gradle Checks" runs-on: block-node-linux-medium From 80a4cac8d3a51c6c6aa9a06154e9a57c6c4e5733 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Fri, 10 Jan 2025 13:20:06 -0700 Subject: [PATCH 27/28] fix: added id-token perm Signed-off-by: Matt Peterson --- .github/workflows/pr-checks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index ada34377a..25c9c40f9 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -17,6 +17,7 @@ env: GRADLE_EXEC: ./gradlew permissions: + id-token: write contents: read jobs: From 662aeb43be8b9feafd0aa7bf951d1901d7d6ccf7 Mon Sep 17 00:00:00 2001 From: Matt Peterson Date: Fri, 10 Jan 2025 13:21:47 -0700 Subject: [PATCH 28/28] fix: added packages write Signed-off-by: Matt Peterson --- .github/workflows/pr-checks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-checks.yaml b/.github/workflows/pr-checks.yaml index 25c9c40f9..2fe5706e9 100644 --- a/.github/workflows/pr-checks.yaml +++ b/.github/workflows/pr-checks.yaml @@ -19,6 +19,7 @@ env: permissions: id-token: write contents: read + packages: write jobs: check-gradle: