From 38ad53d7bfc5eae9bdfe16d1d941d9eb1500711e Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 21 Feb 2020 14:13:31 -0500 Subject: [PATCH] Run build and test jobs in a docker container (#840) (#844) Co-authored-by: btoll --- scripts/release/build/Jenkinsfile | 15 ++++++++++++++- scripts/release/build/stage/build/task.sh | 2 +- scripts/release/build/stage/setup/run.sh | 4 ++-- scripts/release/build/stage/upload/run.sh | 2 +- .../centos.Dockerfile} | 0 scripts/release/common/docker/setup.Dockerfile | 5 +++++ scripts/release/prod/rpm/run_centos.sh | 3 ++- scripts/release/test/Jenkinsfile | 6 +++++- scripts/release/test/rpm/run_centos.sh | 3 ++- 9 files changed, 32 insertions(+), 8 deletions(-) rename scripts/release/common/{centos-build.Dockerfile => docker/centos.Dockerfile} (100%) create mode 100644 scripts/release/common/docker/setup.Dockerfile diff --git a/scripts/release/build/Jenkinsfile b/scripts/release/build/Jenkinsfile index 829c407db0..d5bdde05cb 100644 --- a/scripts/release/build/Jenkinsfile +++ b/scripts/release/build/Jenkinsfile @@ -8,7 +8,20 @@ pipeline { AWS_SECRET_ACCESS_KEY = credentials("prod-s3-aws-secret-access-key") } - agent any + agent { + dockerfile { + filename 'scripts/release/common/docker/setup.Dockerfile' + /* + args '-v /etc/passwd:/etc/passwd' + */ + } + } + + /* + triggers { + cron('* * * * *') + } + */ stages { stage("create ec2 instance") { diff --git a/scripts/release/build/stage/build/task.sh b/scripts/release/build/stage/build/task.sh index 5fbfffafc3..a10955979f 100755 --- a/scripts/release/build/stage/build/task.sh +++ b/scripts/release/build/stage/build/task.sh @@ -31,7 +31,7 @@ else fi # Run RPM build in Centos7 Docker container -sg docker "docker build -t algocentosbuild - < $HOME/go/src/github.com/algorand/go-algorand/scripts/release/common/centos-build.Dockerfile" +sg docker "docker build -t algocentosbuild - < $HOME/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" sg docker "docker run --rm --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/build/rpm/build.sh" diff --git a/scripts/release/build/stage/setup/run.sh b/scripts/release/build/stage/setup/run.sh index 6f6cb59d2d..2cb79accb9 100755 --- a/scripts/release/build/stage/setup/run.sh +++ b/scripts/release/build/stage/setup/run.sh @@ -4,10 +4,10 @@ set -ex # Path(s) are relative to the root of the Jenkins workspace. - INSTANCE=$(cat scripts/release/common/ec2/tmp/instance) +BRANCH="$1" aws s3 cp s3://algorand-devops-misc/tools/gnupg2.2.9_centos7_amd64.tar.bz2 . scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no -r scripts/release/common/setup.sh gnupg2.2.9_centos7_amd64.tar.bz2 ubuntu@"$INSTANCE": -ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash setup.sh "$1" +ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash setup.sh "$BRANCH" diff --git a/scripts/release/build/stage/upload/run.sh b/scripts/release/build/stage/upload/run.sh index a4a617ba29..c96d2d9158 100755 --- a/scripts/release/build/stage/upload/run.sh +++ b/scripts/release/build/stage/upload/run.sh @@ -16,7 +16,7 @@ ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" bash go/src/github.com/ scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no -r ubuntu@"$INSTANCE":~/node_pkg/* pkg/"$FULLVERSION"/ # Create the buildlog file. -scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no ubuntu@"$INSTANCE":~/build_status_"$CHANNEL"_*.asc.gz pkg/ +scp -i ReleaseBuildInstanceKey.pem -o StrictHostKeyChecking=no ubuntu@"$INSTANCE":~/build_status_"$CHANNEL"_*.asc.gz pkg/"$FULLVERSION" #aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --acl public-read pkg/"$FULLVERSION" s3://"$BUCKET_LOCATION"/"$CHANNEL"/"$FULLVERSION"/ aws s3 sync --exclude dev* --exclude master* --exclude nightly* --exclude stable* --acl public-read pkg/"$FULLVERSION" s3://ben-test-2.0.3/"$CHANNEL"/"$FULLVERSION"/ diff --git a/scripts/release/common/centos-build.Dockerfile b/scripts/release/common/docker/centos.Dockerfile similarity index 100% rename from scripts/release/common/centos-build.Dockerfile rename to scripts/release/common/docker/centos.Dockerfile diff --git a/scripts/release/common/docker/setup.Dockerfile b/scripts/release/common/docker/setup.Dockerfile new file mode 100644 index 0000000000..d68331b62c --- /dev/null +++ b/scripts/release/common/docker/setup.Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:18.04 +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y awscli jq ssh +#RUN adduser --uid $(grep jenkins /etc/passwd | awk -F: '{ print $3 }') ubuntu +RUN adduser --uid 111 ubuntu + diff --git a/scripts/release/prod/rpm/run_centos.sh b/scripts/release/prod/rpm/run_centos.sh index 455f10c386..c8a80236e4 100755 --- a/scripts/release/prod/rpm/run_centos.sh +++ b/scripts/release/prod/rpm/run_centos.sh @@ -5,7 +5,8 @@ set -ex . "${HOME}"/build_env -sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/centos-build.Dockerfile" +# Run RPM build in Centos7 Docker container +sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=/run/user/1000/gnupg/S.gpg-agent,dst=/root/S.gpg-agent --mount type=bind,src=${HOME}/prodrepo,dst=/root/prodrepo --mount type=bind,src=${HOME}/keys,dst=/root/keys --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/prod/rpm/snapshot.sh" diff --git a/scripts/release/test/Jenkinsfile b/scripts/release/test/Jenkinsfile index 55e8a3e38c..4b83d7f7e6 100644 --- a/scripts/release/test/Jenkinsfile +++ b/scripts/release/test/Jenkinsfile @@ -12,7 +12,11 @@ pipeline { AWS_SECRET_ACCESS_KEY = credentials("prod-s3-aws-secret-access-key") } - agent any + agent { + dockerfile { + filename 'scripts/release/common/docker/setup.Dockerfile' + } + } stages { stage("create ec2 instance") { diff --git a/scripts/release/test/rpm/run_centos.sh b/scripts/release/test/rpm/run_centos.sh index 2c0f0af2c5..0277bc5e3b 100755 --- a/scripts/release/test/rpm/run_centos.sh +++ b/scripts/release/test/rpm/run_centos.sh @@ -9,7 +9,8 @@ echo date "+build_release begin TEST stage %Y%m%d_%H%M%S" echo -sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/centos-build.Dockerfile" +# Run RPM build in Centos7 Docker container +sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" cat <"${HOME}"/dummyrepo/algodummy.repo [algodummy]