From e2b1d770e49f211dd6170ad2fe00a7156bc20149 Mon Sep 17 00:00:00 2001 From: Oleg Nenashev Date: Mon, 3 Sep 2018 15:01:35 +0200 Subject: [PATCH 1/6] Add a script for launching multiple agents in a burst --- aws/agent-host/launch-agent.sh | 4 +++- aws/agent-host/launch-multiple-agents.sh | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 aws/agent-host/launch-multiple-agents.sh diff --git a/aws/agent-host/launch-agent.sh b/aws/agent-host/launch-agent.sh index 2285288..af995fc 100644 --- a/aws/agent-host/launch-agent.sh +++ b/aws/agent-host/launch-agent.sh @@ -1,10 +1,12 @@ #!/bin/bash +set -xe source "../settings.sh" +AGENT_NUMBER=${1:-0} # Launch worker docker run --rm -d \ --add-host jenkins:${IP_JENKINS_MASTER} --add-host gitserver:${IP_UTIL_CONTAINERS} \ - --name agent -l role=agent \ + --name agent_${AGENT_NUMBER} -l role=agent \ -e "COMMAND_OPTIONS=-master http://jenkins:8080 -executors $(nproc) -description swarm-slave" \ svanoort/jenkins-swarm-agent-mvn:1.0 diff --git a/aws/agent-host/launch-multiple-agents.sh b/aws/agent-host/launch-multiple-agents.sh new file mode 100644 index 0000000..28dfef3 --- /dev/null +++ b/aws/agent-host/launch-multiple-agents.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -xe +AGENT_NUMBER=${1:-5} + +for ((n=0;n<${AGENT_NUMBER};n++)) ; do + bash -ex launch-agent.sh $n +done From 694f6048312c454fe5333145969b53920592b886 Mon Sep 17 00:00:00 2001 From: Oleg Nenashev Date: Mon, 3 Sep 2018 15:31:46 +0200 Subject: [PATCH 2/6] Fix target user to prevent the mess-up --- aws/instance-preconfigure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/instance-preconfigure.sh b/aws/instance-preconfigure.sh index 2d8b6e8..1fe7fdf 100644 --- a/aws/instance-preconfigure.sh +++ b/aws/instance-preconfigure.sh @@ -7,6 +7,6 @@ sudo yum install -y git sudo yum install -y docker && sudo service docker start sudo chkconfig docker on sudo groupadd docker || echo "Group already exists" -sudo usermod -aG docker $USER +sudo usermod -aG docker ec2-user # Add Maven installation? From 79f5bbbc24cd2d07a644079dfeb531f56e948244 Mon Sep 17 00:00:00 2001 From: Oleg Nenashev Date: Mon, 3 Sep 2018 16:30:12 +0200 Subject: [PATCH 3/6] Update test suites for the ARC-530 run --- .../ScriptedCloneAndBuildParallel/Jenkinsfile | 38 +++++++++++++++++++ runner/ARC-530-test.txt | 8 ++++ runner/run-arc-530-test.sh | 2 + 3 files changed, 48 insertions(+) create mode 100644 gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile create mode 100644 runner/ARC-530-test.txt create mode 100755 runner/run-arc-530-test.sh diff --git a/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile b/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile new file mode 100644 index 0000000..08898b9 --- /dev/null +++ b/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile @@ -0,0 +1,38 @@ +def x = [:] +for (def i=0; i<15; i++) { + def j = i; + x.put("branch${i}", { + node { + + // TODO:copy-paste + // Figure out why the Global Tools stuff refuses to work. + // withEnv(["JAVA_HOME=${ tool 'jdk8' }", "PATH+MAVEN=${tool 'maven'}/bin:${env.JAVA_HOME}/bin"]) { + + stage ('Sanity Check') { + echo "Where is maven?" + sh "which mvn" + echo "Where is java?" + sh "which java" + } + + stage('Check out') { + git (credentialsId: "git-ssh", url: "ssh://git@gitserver:2222/git-server/repos/simple-maven-project-with-tests.git") + } + + stage('Build it') { + // Old, fancy way which never worked: + // sh "'${mvnHome}/bin/mvn' -Dmaven.test.failure.ignore clean package" + // New, simple way which works fine: + sh "mvn -Dmaven.test.failure.ignore clean package" + } + + stage('Archive our results') { + junit '**/target/surefire-reports/TEST-*.xml' + archive 'target/*.jar' + } + // } // ends the withEnv block + + }} +} + +parallel(x) \ No newline at end of file diff --git a/runner/ARC-530-test.txt b/runner/ARC-530-test.txt new file mode 100644 index 0000000..f91900a --- /dev/null +++ b/runner/ARC-530-test.txt @@ -0,0 +1,8 @@ +# Trivial testcase for standard Hydra +# Intended to exercise runner framework +# format: test name, jobname, max concurrent number to run, ramp up time in ms, testDuration ms +Sample Test|testcases/LotsOfEchoSteps|6|30000|120000 +Large Shell Step Logs|testcases/LargeShellStepLogs|4|30000|120000 +Massively Parallel Node Use And Sh Steps|testcases/MassivelyParallelNodes|4|30000|450000 +Scripted Pipeline Maven Build - Single|testcases/ScriptedCloneAndBuild|64|600000|300000 +Scripted Pipeline Maven Build - Parallel|testcases/ScriptedCloneAndBuildParallel|64|600000|300000 diff --git a/runner/run-arc-530-test.sh b/runner/run-arc-530-test.sh new file mode 100755 index 0000000..500751e --- /dev/null +++ b/runner/run-arc-530-test.sh @@ -0,0 +1,2 @@ +#!/bin/bash +java -jar target/hydra-runner-1.0-SNAPSHOT-jar-with-dependencies.jar -t "basic-test,sample" -f ARC-530-test.txt -w 30000 -i ${IP_UTIL_CONTAINERS}:8086 -j ${IP_JENKINS_MASTER}:8080 From 6941704db693135531642033b6e0272a59400986 Mon Sep 17 00:00:00 2001 From: Oleg Nenashev Date: Mon, 3 Sep 2018 16:37:49 +0200 Subject: [PATCH 4/6] Fix typo in the script --- gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile b/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile index 08898b9..5b69064 100644 --- a/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile +++ b/gitserver/testcases/ScriptedCloneAndBuildParallel/Jenkinsfile @@ -32,7 +32,7 @@ for (def i=0; i<15; i++) { } // } // ends the withEnv block - }} + }}) } parallel(x) \ No newline at end of file From 82f30eb8cd19154de02ac46a17ac6b3630e032b6 Mon Sep 17 00:00:00 2001 From: Oleg Nenashev Date: Mon, 3 Sep 2018 17:05:13 +0200 Subject: [PATCH 5/6] ARC-530 - Pin versions being used in performance tests --- jenkins/Dockerfile | 2 - jenkins/minimal-plugins.txt | 11 ----- jenkins/plugins.txt | 84 ++++++++++++++++++++++++++----------- 3 files changed, 59 insertions(+), 38 deletions(-) delete mode 100644 jenkins/minimal-plugins.txt diff --git a/jenkins/Dockerfile b/jenkins/Dockerfile index 934ea4f..10906dd 100644 --- a/jenkins/Dockerfile +++ b/jenkins/Dockerfile @@ -33,8 +33,6 @@ COPY minimal-plugins.txt /usr/share/jenkins/ref/minimal-plugins.txt RUN mkdir -p /usr/share/jenkins/ref/plugins \ && echo 'placeholder' > /usr/share/jenkins/ref/plugins/placeholder.lock \ && /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt -RUN echo 'placeholder' > /usr/share/jenkins/ref/plugins/placeholder.lock \ - && /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/minimal-plugins.txt COPY CUSTOM-PLUGINS/*.*pi /usr/share/jenkins/ref/plugins/ # Remove stale executor files and rename HPI files to JPI files to input custom versions diff --git a/jenkins/minimal-plugins.txt b/jenkins/minimal-plugins.txt deleted file mode 100644 index 9b5d3e1..0000000 --- a/jenkins/minimal-plugins.txt +++ /dev/null @@ -1,11 +0,0 @@ -credentials:latest -credentials-binding:latest -metrics:latest -metrics-graphite:latest -random-job-builder:1.0 -ssh-credentials:latest -ssh-slaves:latest -support-core:latest -swarm:3.4 -workflow-durable-task-step:latest -workflow-multibranch:latest \ No newline at end of file diff --git a/jenkins/plugins.txt b/jenkins/plugins.txt index a0ca1cf..9264e25 100644 --- a/jenkins/plugins.txt +++ b/jenkins/plugins.txt @@ -1,26 +1,60 @@ -git:latest -git-client:latest -pipeline-build-step:latest -pipeline-graph-analysis:latest -pipeline-input-step:latest -pipeline-maven:latest -pipeline-milestone-step:latest -pipeline-model-api:latest -pipeline-model-declarative-agent:latest -pipeline-model-definition:latest -pipeline-stage-step:latest -pipeline-stage-tags-metadata:latest -pipeline-stage-view:latest -scm-api:latest -script-security:latest +ace-editor:1.1 +apache-httpcomponents-client-4-api:4.5.5-3.0 +authentication-tokens:1.3 +branch-api:2.0.20 +cloudbees-folder:6.5.1 +command-launcher:1.0 +config-file-provider:2.18 +credentials:2.1.18 +credentials-binding:1.16 +display-url-api:2.2.0 +docker-commons:1.13 +docker-workflow:1.17 +durable-task:1.25 +git:3.9.1 +git-client:2.7.3 +git-server:1.7 +handlebars:1.1.1 +jackson2-api:2.8.11.3 +jdk-tool:1.0 +jquery-detached:1.2.1 +jsch:0.1.54.2 +junit:1.24 +mailer:1.21 +matrix-project:1.13 +metrics:4.0.2.2 +momentjs:1.1.1 +pipeline-build-step:2.7 +pipeline-graph-analysis:1.7 +pipeline-input-step:2.8 +pipeline-maven:3.5.11 +pipeline-milestone-step:1.3.1 +pipeline-model-api:1.3.1 +pipeline-model-declarative-agent:1.1.1 +pipeline-model-definition:1.3.1 +pipeline-model-extensions:1.3.1 +pipeline-rest-api:2.10 +pipeline-stage-step:2.3 +pipeline-stage-tags-metadata:1.3.1 +pipeline-stage-view:2.10 +plain-credentials:1.4 +scm-api:2.2.7 +script-security:1.44 +ssh-credentials:1.14 +ssh-slaves:1.26 +structs:1.14 +support-core:2.49 +swarm:3.4 +token-macro:2.5 +variant:1.1 workflow-aggregator:2.5 -workflow-api:latest -workflow-basic-steps:latest -workflow-cps:latest -workflow-cps-global-lib:latest -workflow-durable-task-step:latest -workflow-job:latest -workflow-multibranch:latest -workflow-scm-step:latest -workflow-step-api:latest -workflow-support:latest \ No newline at end of file +workflow-api:2.29 +workflow-basic-steps:2.9 +workflow-cps:2.54 +workflow-cps-global-lib:2.9 +workflow-durable-task-step:2.20 +workflow-job:2.24 +workflow-multibranch:2.20 +workflow-scm-step:2.6 +workflow-step-api:2.16 +workflow-support:2.20 From 4b909be563e84275a16fbb19096e1b39886aba57 Mon Sep 17 00:00:00 2001 From: Oleg Nenashev Date: Mon, 3 Sep 2018 18:25:04 +0200 Subject: [PATCH 6/6] ARC-530 - Minimal plugin YAGNI --- jenkins/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/Dockerfile b/jenkins/Dockerfile index 10906dd..ad7e8cd 100644 --- a/jenkins/Dockerfile +++ b/jenkins/Dockerfile @@ -29,7 +29,6 @@ ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/full-start.sh"] # Lock files write is a workaround for a bug in plugin install file where it tries to remove nonexistent lock # Basic config - install custom and test framework plugins + custom snapshots COPY plugins.txt /usr/share/jenkins/ref/plugins.txt -COPY minimal-plugins.txt /usr/share/jenkins/ref/minimal-plugins.txt RUN mkdir -p /usr/share/jenkins/ref/plugins \ && echo 'placeholder' > /usr/share/jenkins/ref/plugins/placeholder.lock \ && /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt