From 566405e0c6770e9bc675d5791d1d8388bc9598d1 Mon Sep 17 00:00:00 2001 From: btoll Date: Fri, 30 Oct 2020 15:58:27 -0400 Subject: [PATCH] Fix bugs in mule test stage (#1679) Miscellaneous sign and test pipeline fixes: - fix status file name - change name of image in mule yaml - add vars to dynamically account for both algorand and algorand-devtools packages (all channels) --- package-sign.yaml | 2 +- scripts/release/mule/sign/sign.sh | 3 +- scripts/release/mule/test/test.sh | 36 +++++++++++++++++-- .../tests/post/verify_package_binaries.sh | 8 ++--- .../test/tests/pre/verify_control_files.sh | 20 ++++++----- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/package-sign.yaml b/package-sign.yaml index be567849c1..fc3674f992 100644 --- a/package-sign.yaml +++ b/package-sign.yaml @@ -1,7 +1,7 @@ agents: - name: deb dockerFilePath: docker/build/cicd.ubuntu.Dockerfile - image: algorand/cicd-ubuntu + image: algorand/go-algorand-ci-linux-ubuntu version: scripts/configure_dev-deps.sh buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` diff --git a/scripts/release/mule/sign/sign.sh b/scripts/release/mule/sign/sign.sh index f1dd9475dd..7fec7f677d 100755 --- a/scripts/release/mule/sign/sign.sh +++ b/scripts/release/mule/sign/sign.sh @@ -14,7 +14,6 @@ BRANCH=${BRANCH:-$(./scripts/compute_branch.sh)} CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")} PKG_DIR="./tmp/node_pkgs" SIGNING_KEY_ADDR=dev@algorand.com -STATUSFILE="build_status_${CHANNEL}_${VERSION}" # It seems that copying/mounting the gpg dir from another machine can result in insecure # access privileges, so set the correct permissions to avoid the following warning: @@ -62,7 +61,6 @@ for os in "${OS_TYPES[@]}"; do done HASHFILE="hashes_${CHANNEL}_${os}_${arch}_${VERSION}" - md5sum *.tar.gz *.deb *.rpm >> "$HASHFILE" shasum -a 256 *.tar.gz *.deb *.rpm >> "$HASHFILE" shasum -a 512 *.tar.gz *.deb *.rpm >> "$HASHFILE" @@ -70,6 +68,7 @@ for os in "${OS_TYPES[@]}"; do gpg -u "$SIGNING_KEY_ADDR" --detach-sign "$HASHFILE" gpg -u "$SIGNING_KEY_ADDR" --clearsign "$HASHFILE" + STATUSFILE="build_status_${CHANNEL}_${os}-${arch}_${VERSION}" gpg -u "$SIGNING_KEY_ADDR" --clearsign "$STATUSFILE" gzip -c "$STATUSFILE.asc" > "$STATUSFILE.asc.gz" ) diff --git a/scripts/release/mule/test/test.sh b/scripts/release/mule/test/test.sh index ff96ebe857..33b80c76d5 100755 --- a/scripts/release/mule/test/test.sh +++ b/scripts/release/mule/test/test.sh @@ -16,9 +16,14 @@ export CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")} export NETWORK=${NETWORK:-$(./scripts/compute_branch_network.sh "$BRANCH")} export SHA=${SHA:-$(git rev-parse HEAD)} export VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)} +ALGORAND_PACKAGE_NAME=$([ "$CHANNEL" = beta ] && echo algorand-beta || echo algorand) +DEVTOOLS_PACKAGE_NAME=$([ "$CHANNEL" = beta ] && echo algorand-devtools-beta || echo algorand-devtools) +export ALGORAND_PACKAGE_NAME +export DEVTOOLS_PACKAGE_NAME PKG_DIR="./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE" +mkdir -p "$PKG_DIR" pushd "$PKG_DIR" if [ -n "$S3_SOURCE" ] @@ -38,6 +43,7 @@ popd for test in $(ls ./scripts/release/mule/test/tests/pre/*.sh) do + echo ">>>>>>>>>> PRE TESTING $(basename "$test")" bash "$test" done @@ -52,14 +58,40 @@ else # Normally, this is installed for us b/c it's a dependency. # See `./installer/rpm/algorand/algorand.spec`. yum install yum-cron -y - rpm -i algorand-"$VERSION"-1."$ARCH_BIT".rpm - rpm -i algorand-devtools-"$VERSION"-1."$ARCH_BIT".rpm + # + # Note that the RPM package DOES NOT have the CHANNEL in its filename (unlike DEB), + # instead it contains the package name. + # + # deb: + # algorand_CHANNEL*VERSION.deb + # algorand-devtools_CHANNEL*VERSION.deb + # + # (this pattern is for all channels) + # + # rpm: + # (this pattern is for stable) + # algorand-VERSION*.rpm + # algorand-devtools-VERSION.rpm + # + # (this pattern is for beta) + # algorand-beta-VERSION*.rpm + # algorand-devtools-beta-VERSION.rpm + # + # SO..... + # ALGORAND_PACKAGE_NAME-VERSION*.rpm + # DEVTOOLS_PACKAGE_NAME-beta-VERSION.rpm + # + # Hope that makes sense :) + # + rpm -i "$ALGORAND_PACKAGE_NAME"-"$VERSION"-1."$ARCH_BIT".rpm + rpm -i "$DEVTOOLS_PACKAGE_NAME"-*"$VERSION"-1."$ARCH_BIT".rpm fi popd for test in $(ls ./scripts/release/mule/test/tests/post/*.sh) do + echo ">>>>>>>>>> POST TESTING $(basename "$test")" bash "$test" done diff --git a/scripts/release/mule/test/tests/post/verify_package_binaries.sh b/scripts/release/mule/test/tests/post/verify_package_binaries.sh index de9abb56c1..1a61dba1e7 100755 --- a/scripts/release/mule/test/tests/post/verify_package_binaries.sh +++ b/scripts/release/mule/test/tests/post/verify_package_binaries.sh @@ -12,11 +12,11 @@ cd "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE" if [ "$PKG_TYPE" = deb ] then - dpkg -L algorand > "$RPMTMP/algorand.install" - dpkg -L algorand-devtools > "$RPMTMP/algorand-devtools.install" + dpkg -L "$ALGORAND_PACKAGE_NAME" > "$RPMTMP/algorand.install" + dpkg -L "$DEVTOOLS_PACKAGE_NAME" > "$RPMTMP/algorand-devtools.install" else - rpm -ql algorand > "$RPMTMP/algorand.install" - rpm -ql algorand-devtools > "$RPMTMP/algorand-devtools.install" + rpm -ql "$ALGORAND_PACKAGE_NAME" > "$RPMTMP/algorand.install" + rpm -ql "$DEVTOOLS_PACKAGE_NAME" > "$RPMTMP/algorand-devtools.install" fi ALGORAND_BINS=( diff --git a/scripts/release/mule/test/tests/pre/verify_control_files.sh b/scripts/release/mule/test/tests/pre/verify_control_files.sh index adb7c98209..a4c6c3e30c 100755 --- a/scripts/release/mule/test/tests/pre/verify_control_files.sh +++ b/scripts/release/mule/test/tests/pre/verify_control_files.sh @@ -15,24 +15,26 @@ then # Pre-Depends: algorand (>= 2.1.6) # - cp "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"/algorand-devtools*"$VERSION"*.deb "$RPMTMP" + cp "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"/algorand-devtools*"$CHANNEL"*"$VERSION"*.deb "$RPMTMP" cd "$RPMTMP" ar xv *"$VERSION"*.deb tar xf control.tar.xz - if ! grep -F "Pre-Depends: algorand (>= $VERSION)" control + if ! grep -F "Pre-Depends: $ALGORAND_PACKAGE_NAME (>= $VERSION)" control then - echo "[$0] The dependency for algorand version $VERSION is incorrect." + echo "[$0] The dependency for $ALGORAND_PACKAGE_NAME version $VERSION is incorrect." exit 1 fi - echo "[$0] The dependency for algorand version $VERSION is correct." + echo "[$0] The dependency for $ALGORAND_PACKAGE_NAME version $VERSION is correct." else # Note that the .spec file isn't packaged in the RPM. There are tools such `rpmrebuild` that # attempt to generate the .spec file, but it doesn't give us the info we need. # # Instead, we'll just install using `dpkg` and grep the error stream. - if ! rpm -i "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE/algorand-devtools-$VERSION"*"$ARCH_BIT".rpm 2> "$RPMTMP/rpm.install" + # + # Also, note that the RPM package DOES NOT have the CHANNEL in its filename (unlike DEB)!! + if ! rpm -i "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE/algorand-devtools-"*"$VERSION"*"$ARCH_BIT".rpm 2> "$RPMTMP/rpm.install" then # # We're looking for lines that looks like the following: @@ -40,16 +42,16 @@ else # error: Failed dependencies: # algorand >= 2.1.86017 is needed by algorand-devtools-2.1.86017-1.x86_64 # - if [[ $(cat "$RPMTMP/rpm.install") =~ "algorand >= $VERSION is needed by algorand-devtools-$VERSION" ]] + if [[ $(cat "$RPMTMP/rpm.install") =~ "$ALGORAND_PACKAGE_NAME >= $VERSION is needed by $DEVTOOLS_PACKAGE_NAME-$VERSION" ]] then - echo "[$0] The package \`algorand-devtools\` correctly has a dependency on package \`algorand\` and failed to install." + echo "[$0] The package \`algorand-devtools\` correctly has a dependency on package $ALGORAND_PACKAGE_NAME and failed to install." exit 0 fi - echo "[$0] The package \`algorand-devtools\` failed to install because of a missing dependency other than the \`algorand\` package." + echo "[$0] The package \`algorand-devtools\` failed to install because of a missing dependency other than the $ALGORAND_PACKAGE_NAME package." exit 1 else - echo "[$0] The package \`algorand-devtools\` was installed without any dependencies, while it should have a dependency on the \`algorand\` package." + echo "[$0] The package \`algorand-devtools\` was installed without any dependencies, while it should have a dependency on the $ALGORAND_PACKAGE_NAME package." exit 1 fi fi