From e563a41a921e4077b1abf40e6821aeb55b463d4e Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Thu, 1 Jun 2017 16:46:50 +0200 Subject: [PATCH 01/12] geocat-dev - first shot on CI materials --- Jenkinsfile | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ web/pom.xml | 27 ++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..c763311b11 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,53 @@ +#!/usr/bin/groovy + +@Library('c2c-pipeline-library') import static com.camptocamp.utils.* + + +dockerBuild { + stage('Docker pull the maven image') { + sh 'docker pull maven:3-jdk-8' + sh 'docker pull pmauduit/google-drive-publisher' + } + withDockerContainer(image: 'maven:3-jdk-8') { + stage('Getting the sources') { + git url: 'git@github.com:camptocamp/geocat.git', branch: env.BRANCHE_NAME + sh 'git submodule update --init --recursive' + } + stage('First build without test') { + sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -DskipTests''' + } + stage('Second build with tests') { + sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -fn''' + } + stage('Saving tests results') { + junit '**/target/surefire-reports/TEST-*.xml' + } + stage('configure georchestra c2c docker-hub account') { + // Requires a username / password configured in Jenkins' credentials, with id docker-c2cgeorchestra + withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'dockerhub', + usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) { + def configXmlStr = """ + + + docker-hub + USERNAME + PASSWORD + + geocat2@camptocamp.com + + + + """.replaceAll("USERNAME", env.USERNAME).replaceAll("PASSWORD", env.PASSWORD) + sh "echo '${configXmlStr}' > /config.xml" + } + } + stage('Build/publish a docker image') { + // one-liner to setup docker + sh 'curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz && tar --strip-components=1 -xvzf docker-17.05.0-ce.tgz -C /usr/local/bin' + sh '''mvn -s /settings.xml -B -Dmaven.repo.local=./.m2_repo -pl web -Pdocker docker:build docker:push''' + } + } // withDockerContainer +} diff --git a/web/pom.xml b/web/pom.xml index 309a32bcec..1723979c43 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -793,8 +793,33 @@ es + + docker + + + + com.spotify + docker-maven-plugin + 0.4.14 + + camptocamp/geocat:${project.version} + tomcat:8-jre8 + + + /usr/local/tomcat/webapps + ${project.build.directory} + geonetwork.war + + + + docker-hub + https://index.docker.io/v1/ + + + + + - ${project.version} SNAPSHOT From 275dd66438ee998f81b9b37ea7afd0dab3e0fa33 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Fri, 2 Jun 2017 13:40:38 +0200 Subject: [PATCH 02/12] ci - changing repository url --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c763311b11..3155fb5ed8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ dockerBuild { } withDockerContainer(image: 'maven:3-jdk-8') { stage('Getting the sources') { - git url: 'git@github.com:camptocamp/geocat.git', branch: env.BRANCHE_NAME + git url: 'https://github.com/geoadmin/geocat.git', branch: "geocat_3.4.x" sh 'git submodule update --init --recursive' } stage('First build without test') { From b2f5c2cb252e648f1249b1b11a38c63ee012616f Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 5 Jun 2017 17:49:21 +0200 Subject: [PATCH 03/12] CI - using the current branch instead of 3.0.4 --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3155fb5ed8..14708c3991 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,12 +6,11 @@ dockerBuild { stage('Docker pull the maven image') { sh 'docker pull maven:3-jdk-8' - sh 'docker pull pmauduit/google-drive-publisher' } withDockerContainer(image: 'maven:3-jdk-8') { stage('Getting the sources') { - git url: 'https://github.com/geoadmin/geocat.git', branch: "geocat_3.4.x" - sh 'git submodule update --init --recursive' + git url: 'https://github.com/geoadmin/geocat.git', branch: env.BRANCH_NAME + sh 'git submodule update --init --recursive' } stage('First build without test') { sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -DskipTests''' From 79f227a1318baf367e11ac45e277ea5e3fcc673d Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 5 Jun 2017 17:53:43 +0200 Subject: [PATCH 04/12] CI - s/geoadmin/camptocamp/ --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 14708c3991..27a3ce9b42 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ dockerBuild { } withDockerContainer(image: 'maven:3-jdk-8') { stage('Getting the sources') { - git url: 'https://github.com/geoadmin/geocat.git', branch: env.BRANCH_NAME + git url: 'https://github.com/camptocamp/geocat.git', branch: env.BRANCH_NAME sh 'git submodule update --init --recursive' } stage('First build without test') { From b7dd6b1fca8924e598b5928a192b9871421f1826 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Tue, 6 Jun 2017 11:16:23 +0200 Subject: [PATCH 05/12] CI - fixing filename mismatch --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 27a3ce9b42..6d0f56cb26 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ dockerBuild { """.replaceAll("USERNAME", env.USERNAME).replaceAll("PASSWORD", env.PASSWORD) - sh "echo '${configXmlStr}' > /config.xml" + sh "echo '${configXmlStr}' > /settings.xml" } } stage('Build/publish a docker image') { From d73994059a23af6c2b79beeea0e96e967506de9f Mon Sep 17 00:00:00 2001 From: Marc Sutter Date: Wed, 7 Jun 2017 14:10:25 +0200 Subject: [PATCH 06/12] Add filter for xlarge instance of slaves --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 6d0f56cb26..72edd3820f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,6 +2,9 @@ @Library('c2c-pipeline-library') import static com.camptocamp.utils.* +selectNodes { + (it.memorysize_mb as Float) > 12000 +} dockerBuild { stage('Docker pull the maven image') { From ea586d62ff5b82596323a2e3140d5ff609cc1360 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 12 Jun 2017 11:58:18 +0200 Subject: [PATCH 07/12] CI - Adding logic to autodeploy on caas.dev.bgdi.ch --- Jenkinsfile | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 72edd3820f..513dbe1ec0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,4 +52,56 @@ dockerBuild { sh '''mvn -s /settings.xml -B -Dmaven.repo.local=./.m2_repo -pl web -Pdocker docker:build docker:push''' } } // withDockerContainer + + // Using another container, deploys the previously published image onto the dev env + stage('Deploy newly created images on the dev env') { + withDockerContainer(image: 'debian', args: "--privileged -u 0:0") { + + stage('Install / configure needed tools') { + sh 'apt update && apt install -y make ssh git wget unzip' + sh 'mkdir -p ~/bin' + } // stage + + stage("Prepare caas-dev access") { + withCredentials([file(credentialsId: 'jenkins-caas-dev-bgdi.ch.json', variable: 'FILE')]) { + sh 'mkdir -p ~/.rancher' + sh "cp ${FILE} ~/.rancher/caas.dev.bgdi.ch.json" + } // withCredentials + } // stage + + stage("Configuring AWS / S3") { + sh 'mkdir ~/.aws' + withCredentials([[$class: 'UsernamePasswordMultiBinding', + credentialsId: 'terraform-georchestra-aws-credentials', + usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) { + def credentialsFile = """ +[c2c] +aws_access_key_id = ${env.USERNAME} +aws_secret_access_key = ${env.PASSWORD} +region = eu-west-1 +""" + sh "echo '${credentialsFile}' > ~/.aws/credentials" + } // withCredentials + } // stage + + stage('Checking out the terraform-geocat repository') { + sshagent(["terraform-geocat-deploy-key"]) { + sh "ssh -oStrictHostKeyChecking=no git@github.com || true" + sh "rm -rf terraform-geocat" + sh "git clone git@github.com:camptocamp/terraform-geocat.git" + } // sshagent + } // stage + + stage('Terraforming') { + ansiColor('xterm') { + sh """cd terraform-geocat && + ln -s /root/bin/terraform /usr/bin && + make install && + make init && + cd rancher-environments/geocat-dev && + terraform apply""" + } // ansiColor + } // stage + } // withDockerContainer + } // stage } From 0dbdb5c9ad9c81ae21542303f2bdf7d7afa40d02 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 12 Jun 2017 13:36:26 +0200 Subject: [PATCH 08/12] CI - conditional redeploy I don't really know if it will work this way, just trying ... --- Jenkinsfile | 94 +++++++++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 513dbe1ec0..3b5c444247 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,54 +54,56 @@ dockerBuild { } // withDockerContainer // Using another container, deploys the previously published image onto the dev env - stage('Deploy newly created images on the dev env') { - withDockerContainer(image: 'debian', args: "--privileged -u 0:0") { + if (env.BRANCH_NAME == 'geocat_3.4.x') { + stage('Deploy newly created images on the dev env') { + withDockerContainer(image: 'debian', args: "--privileged -u 0:0") { - stage('Install / configure needed tools') { - sh 'apt update && apt install -y make ssh git wget unzip' - sh 'mkdir -p ~/bin' - } // stage + stage('Install / configure needed tools') { + sh 'apt update && apt install -y make ssh git wget unzip' + sh 'mkdir -p ~/bin' + } // stage - stage("Prepare caas-dev access") { - withCredentials([file(credentialsId: 'jenkins-caas-dev-bgdi.ch.json', variable: 'FILE')]) { - sh 'mkdir -p ~/.rancher' - sh "cp ${FILE} ~/.rancher/caas.dev.bgdi.ch.json" - } // withCredentials - } // stage + stage("Prepare caas-dev access") { + withCredentials([file(credentialsId: 'jenkins-caas-dev-bgdi.ch.json', variable: 'FILE')]) { + sh 'mkdir -p ~/.rancher' + sh "cp ${FILE} ~/.rancher/caas.dev.bgdi.ch.json" + } // withCredentials + } // stage - stage("Configuring AWS / S3") { - sh 'mkdir ~/.aws' - withCredentials([[$class: 'UsernamePasswordMultiBinding', - credentialsId: 'terraform-georchestra-aws-credentials', - usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) { - def credentialsFile = """ -[c2c] -aws_access_key_id = ${env.USERNAME} -aws_secret_access_key = ${env.PASSWORD} -region = eu-west-1 -""" - sh "echo '${credentialsFile}' > ~/.aws/credentials" - } // withCredentials - } // stage + stage("Configuring AWS / S3") { + sh 'mkdir ~/.aws' + withCredentials([[$class: 'UsernamePasswordMultiBinding', + credentialsId: 'terraform-georchestra-aws-credentials', + usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) { + def credentialsFile = """ + [c2c] + aws_access_key_id = ${env.USERNAME} + aws_secret_access_key = ${env.PASSWORD} + region = eu-west-1 + """ + sh "echo '${credentialsFile}' > ~/.aws/credentials" + } // withCredentials + } // stage - stage('Checking out the terraform-geocat repository') { - sshagent(["terraform-geocat-deploy-key"]) { - sh "ssh -oStrictHostKeyChecking=no git@github.com || true" - sh "rm -rf terraform-geocat" - sh "git clone git@github.com:camptocamp/terraform-geocat.git" - } // sshagent - } // stage + stage('Checking out the terraform-geocat repository') { + sshagent(["terraform-geocat-deploy-key"]) { + sh "ssh -oStrictHostKeyChecking=no git@github.com || true" + sh "rm -rf terraform-geocat" + sh "git clone git@github.com:camptocamp/terraform-geocat.git" + } // sshagent + } // stage - stage('Terraforming') { - ansiColor('xterm') { - sh """cd terraform-geocat && - ln -s /root/bin/terraform /usr/bin && - make install && - make init && - cd rancher-environments/geocat-dev && - terraform apply""" - } // ansiColor - } // stage - } // withDockerContainer - } // stage -} + stage('Terraforming') { + ansiColor('xterm') { + sh """cd terraform-geocat && + ln -s /root/bin/terraform /usr/bin && + make install && + make init && + cd rancher-environments/geocat-dev && + terraform apply""" + } // ansiColor + } // stage + } // withDockerContainer + } // stage + } // if +} // dockerBuild From 3810f992bd638d3ccbb75cd2a25242b1e7a35b31 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 12 Jun 2017 14:13:33 +0200 Subject: [PATCH 09/12] CI - configure the dev env appropriately --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3b5c444247..f53c54a50e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,10 +16,10 @@ dockerBuild { sh 'git submodule update --init --recursive' } stage('First build without test') { - sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -DskipTests''' + sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork -DskipTests''' } stage('Second build with tests') { - sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -fn''' + sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork -fn''' } stage('Saving tests results') { junit '**/target/surefire-reports/TEST-*.xml' @@ -49,7 +49,7 @@ dockerBuild { stage('Build/publish a docker image') { // one-liner to setup docker sh 'curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz && tar --strip-components=1 -xvzf docker-17.05.0-ce.tgz -C /usr/local/bin' - sh '''mvn -s /settings.xml -B -Dmaven.repo.local=./.m2_repo -pl web -Pdocker docker:build docker:push''' + sh '''mvn -s /settings.xml -B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork -pl web -Pdocker docker:build docker:push''' } } // withDockerContainer @@ -95,7 +95,7 @@ dockerBuild { stage('Terraforming') { ansiColor('xterm') { - sh """cd terraform-geocat && + sh """cd terraform-geocat && ln -s /root/bin/terraform /usr/bin && make install && make init && From 0cb180c13d096a8444e33ee912b102a4d6b446be Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 12 Jun 2017 14:47:52 +0200 Subject: [PATCH 10/12] CI - if statement does not work, see build 19. Trying to move it into a stage --- Jenkinsfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f53c54a50e..6cd19e9773 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,7 +54,6 @@ dockerBuild { } // withDockerContainer // Using another container, deploys the previously published image onto the dev env - if (env.BRANCH_NAME == 'geocat_3.4.x') { stage('Deploy newly created images on the dev env') { withDockerContainer(image: 'debian', args: "--privileged -u 0:0") { @@ -95,15 +94,18 @@ dockerBuild { stage('Terraforming') { ansiColor('xterm') { - sh """cd terraform-geocat && - ln -s /root/bin/terraform /usr/bin && - make install && - make init && - cd rancher-environments/geocat-dev && - terraform apply""" + if (env.BRANCH_NAME == 'geocat_3.4.x') { + sh """cd terraform-geocat && + ln -s /root/bin/terraform /usr/bin && + make install && + make init && + cd rancher-environments/geocat-dev && + terraform apply""" + } else { + println "Not onto the 'geocat_3.4.x' branch, skipping redeploy" + }// if } // ansiColor } // stage } // withDockerContainer } // stage - } // if } // dockerBuild From 51837b55aceec662c505bc7fd5ceb392e191efa5 Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Mon, 12 Jun 2017 18:32:25 +0200 Subject: [PATCH 11/12] CI - refactoring maven options, adding coverage info --- Jenkinsfile | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6cd19e9773..0731a925b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,6 +7,9 @@ selectNodes { } dockerBuild { + + def mavenOpts = '-B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork' + stage('Docker pull the maven image') { sh 'docker pull maven:3-jdk-8' } @@ -16,10 +19,24 @@ dockerBuild { sh 'git submodule update --init --recursive' } stage('First build without test') { - sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork -DskipTests''' + sh "mvn clean install ${mavenOpts} -DskipTests" } stage('Second build with tests') { - sh '''mvn clean install -B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork -fn''' + sh "mvn clean install ${mavenOpts} -fn" + } + stage('calculating coverage') { + sh "mvn cobertura:cobertura ${mavenOpts} -fn" + step([$class: 'CoberturaPublisher', + autoUpdateHealth: false, + autoUpdateStability: false, + coberturaReportFile: '**/target/site/cobertura/coverage.xml', + failNoReports: true, + failUnhealthy: false, + failUnstable: false, + maxNumberOfBuilds: 0, + onlyStable: false, + sourceEncoding: 'UTF_8', + zoomCoverageChart: true]) } stage('Saving tests results') { junit '**/target/surefire-reports/TEST-*.xml' @@ -49,7 +66,7 @@ dockerBuild { stage('Build/publish a docker image') { // one-liner to setup docker sh 'curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz && tar --strip-components=1 -xvzf docker-17.05.0-ce.tgz -C /usr/local/bin' - sh '''mvn -s /settings.xml -B -Dmaven.repo.local=./.m2_repo -Ddb.username=geonetwork -Ddb.name=geonetwork -Ddb.type=postgres -Ddb.host=database -Ddb.password=geonetwork -pl web -Pdocker docker:build docker:push''' + sh "mvn -s /settings.xml ${mavenOpts} -pl web -Pdocker docker:build docker:push" } } // withDockerContainer From 79ef3d1b77a1e4d8c41eb17eb0c258b760c1de4f Mon Sep 17 00:00:00 2001 From: Pierre Mauduit Date: Tue, 13 Jun 2017 10:22:16 +0200 Subject: [PATCH 12/12] CI - forces cobertura's output as XML --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0731a925b8..ebfdd5332f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ dockerBuild { sh "mvn clean install ${mavenOpts} -fn" } stage('calculating coverage') { - sh "mvn cobertura:cobertura ${mavenOpts} -fn" + sh "mvn cobertura:cobertura ${mavenOpts} -fn -Dcobertura.report.format=xml" step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false,