From bec6008c9f14253d1b62e82a6d529930c161adfb Mon Sep 17 00:00:00 2001 From: Stefan Ranoszek Date: Tue, 3 Dec 2024 13:11:21 +0000 Subject: [PATCH 1/6] feat: optimise build process --- .github/workflows/docker.yaml | 11 ++- .github/workflows/e2etest.yaml | 7 +- .gitignore | 3 + infra/docker/batch/Dockerfile | 47 +++++---- pom.xml | 59 ++++++++++-- src/main/java/AppJarRunner.java | 162 ++++++++++++++++++++++++++++++++ 6 files changed, 256 insertions(+), 33 deletions(-) create mode 100644 src/main/java/AppJarRunner.java diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index d7593e34c..cec3480d3 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -51,7 +51,7 @@ jobs: proxies: '[{ "id":"dvsa-ci-proxy", "active": "true", "protocol": "http", "host": "${{vars.CI_PROXY}}", "port": "${{vars.CI_PROXYPORT}}" }]' repositories: '[ { "id":"dvsa-github-packages", "name":"dvsa-github-packages", "url":"https://maven.pkg.github.com/dvsa/*", "releases": { "enabled": "true" }, "snapshots": { "enabled": "true" } }]' output_file: ./settings.xml - + - name: view settings.xml run: cat ./settings.xml @@ -60,7 +60,6 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v5 with: images: | @@ -96,12 +95,12 @@ jobs: if: ${{ inputs.push }} run: | aws ecr batch-delete-image --repository-name vol-qa/vft --image-ids imageTag=latest - + - name: Build ${{ inputs.push && 'and push ' || '' }}Docker image id: build-and-push uses: docker/build-push-action@v5 with: - context: . + context: . file: infra/docker/batch/Dockerfile platforms: linux/amd64,linux/arm64 push: ${{ inputs.push }} @@ -109,7 +108,9 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - + + - name: Remove settings.xml + run: rm -f ./settings.xml - name: Setup Notation CLI if: ${{ inputs.push }} diff --git a/.github/workflows/e2etest.yaml b/.github/workflows/e2etest.yaml index 3f7e93012..05aef9b36 100644 --- a/.github/workflows/e2etest.yaml +++ b/.github/workflows/e2etest.yaml @@ -104,9 +104,10 @@ jobs: --timeout "attemptDurationSeconds=$(( ${{ inputs.batch_timeout_minutes }} * 60 ))" \ --container-overrides '{ "command": [ - "/bin/sh", - "-c", - "./run.sh" + "java", + "-cp", + "app.jar" + "AppJarRunner" ], "environment": [ { diff --git a/.gitignore b/.gitignore index 9ebb678d8..e471c66f8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ *.tar.gz *.rar +# Ignore Maven settings file +settings.xml + # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* diff --git a/infra/docker/batch/Dockerfile b/infra/docker/batch/Dockerfile index 19450d9e4..67199e694 100644 --- a/infra/docker/batch/Dockerfile +++ b/infra/docker/batch/Dockerfile @@ -1,30 +1,45 @@ -FROM maven:3.9.6-amazoncorretto-11 as maven +FROM maven:3.9.6-amazoncorretto-11 AS builder +WORKDIR /build -ENV platform="LINUX" -ENV gridURL="https://selenium-hub.olcs.dev-dvsacloud.uk/" +COPY pom.xml . +COPY settings.xml /root/.m2/settings.xml +COPY src ./src +RUN mvn clean package -DskipTests +FROM amazoncorretto:11 -ENV proxyHost=$proxyHost -ENV proxyPort=$proxyPort +# hadolint ignore=DL3044 +ENV JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto \ + MAVEN_HOME=/usr/share/maven \ + PATH=${JAVA_HOME}/bin:/usr/share/maven/bin:${PATH} WORKDIR /app RUN yum update -y && \ - yum install -y zip-3.* unzip-6.* && \ + yum install -y \ + zip-3.* \ + unzip-6.* \ + curl-7.* \ + python3-3.* && \ yum clean all && \ - rm -rf /var/cache/yum + rm -rf /var/cache/yum && \ + python3 -m pip install --no-cache-dir awscli==1.27.165 -# Install latest version of the aws cli -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ - ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update && \ - rm -f awscliv2.zip +COPY --from=builder /usr/share/maven /usr/share/maven -COPY . . +RUN mkdir -p /root/.m2 +COPY settings.xml /root/.m2/settings.xml +COPY --from=builder /root/.m2/repository /root/.m2/repository -COPY ./settings.xml /root/.m2/settings.xml +RUN ls -la /usr/share/maven/bin/mvn && \ + chmod +x /usr/share/maven/bin/mvn +COPY pom.xml ./pom.xml +COPY src ./src -RUN chmod +x ./run.sh +COPY --from=builder /build/target/*.jar ./app.jar -CMD ["/bin/sh", "-c", "./run.sh"] \ No newline at end of file +# Remove settings.xml after use +RUN rm -f /root/.m2/settings.xml + +ENTRYPOINT ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/pom.xml b/pom.xml index aabd1f8a9..d68e9eb65 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ 3.2.0 3.2.5 3.2.5 + 3.2.4 1.10.0 1.17.1 2.10.1 @@ -254,21 +255,61 @@ - io.qameta.allure - allure-maven - 2.12.0 + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + + + AppJarRunner + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 - ${project.basedir}/target/site/allure-report - - ${allure-report.version} + + + true + AppJarRunner + + + + + org.apache.maven.plugins + maven-dependency-plugin - generate-test-report - verify + copy-dependencies + prepare-package - report + copy-dependencies + + ${project.build.directory}/dependency + diff --git a/src/main/java/AppJarRunner.java b/src/main/java/AppJarRunner.java new file mode 100644 index 000000000..d9080c1e3 --- /dev/null +++ b/src/main/java/AppJarRunner.java @@ -0,0 +1,162 @@ +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class AppJarRunner { + private static final int MAX_RETRIES = 1; + + public static void main(String[] args) { + try { + setSystemProperties(); + runTestsWithRetry(); + generateReports(); + } catch (Exception e) { + System.err.println("An error occurred while running the tests: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + private static void runTestsWithRetry() throws Exception { + boolean success = false; + int attempt = 0; + + while (!success && attempt <= MAX_RETRIES) { + try { + if (attempt > 0) { + System.out.println("Retry attempt " + attempt + " for failed tests"); + } + List command = buildMavenCommand(attempt > 0); + success = executeCommand(command); + if (success) break; + } catch (Exception e) { + if (attempt == MAX_RETRIES) { + throw e; + } + } + attempt++; + } + + if (!success) { + throw new Exception("Tests failed after " + (MAX_RETRIES + 1) + " attempts"); + } + } + + private static void setSystemProperties() { + setAndLogProperty("wdm.proxy", String.format("%s:%s", + System.getenv("proxyHost"), System.getenv("proxyPort"))); + setAndLogProperty("https.proxyHost", System.getenv("proxyHost")); + setAndLogProperty("https.proxyPort", System.getenv("proxyPort")); + setAndLogProperty("http.proxyHost", System.getenv("proxyHost")); + setAndLogProperty("http.proxyPort", System.getenv("proxyPort")); + setAndLogProperty("http.nonProxyHosts", System.getenv("noProxyJava")); + setAndLogProperty("env", System.getenv("platformEnv")); + setAndLogProperty("browser", System.getenv("browserName")); + setAndLogProperty("browserVersion", System.getenv("browserVersion")); + setAndLogProperty("platform", System.getenv("platform")); + setAndLogProperty("gridURL", System.getenv("gridURL")); + setAndLogProperty("tag.name", "(not " + System.getenv("exclude_tags") + ")"); + setAndLogProperty("cucumber.filter.tags", System.getenv("cucumberTags")); + } + + private static void setAndLogProperty(String key, String value) { + if (value != null) { + System.setProperty(key, value); + } + } + + private static boolean executeCommand(List command) throws Exception { + System.out.println("Executing command: " + String.join(" ", command)); + + String mavenPath = System.getenv("MAVEN_HOME") + "/bin/mvn"; + File mavenFile = new File(mavenPath); + if (!mavenFile.exists()) { + throw new Exception("Maven not found at " + mavenPath); + } + + command.set(0, mavenPath); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.inheritIO(); + Process process = processBuilder.start(); + return process.waitFor() == 0; + } + + private static List buildMavenCommand(boolean isRetry) { + List command = new ArrayList<>(Arrays.asList( + "mvn", "--batch-mode", + isRetry ? "verify" : "clean verify", + "-fae", "-U", + "-Denv=" + System.getenv("platformEnv"), + "-DgridURL=" + System.getenv("gridURL"), + "-Dbrowser=" + System.getenv("browserName"), + "-DbrowserVersion=" + System.getenv("browserVersion"), + "-Dplatform=" + System.getenv("platform") + )); + + if (isRetry) { + command.add("-Dfailsafe.rerunFailingTestsCount=1"); + command.add("-Dsurefire.rerunFailingTestsCount=1"); + } + + String mavenOptions = System.getenv("mavenOptions"); + if (mavenOptions != null && !mavenOptions.isEmpty()) { + command.addAll(Arrays.asList(mavenOptions.split("\\s+"))); + } else { + command.add("-Dcucumber.options=-- io.qameta.allure.cucumber7jvm.AllureCucumber7Jvm"); + } + + return command; + } + + private static void generateReports() throws Exception { + List reportCommand = Arrays.asList( + System.getenv("MAVEN_HOME") + "/bin/mvn", + "allure:report" + ); + + ProcessBuilder reportBuilder = new ProcessBuilder(reportCommand); + reportBuilder.inheritIO(); + Process reportProcess = reportBuilder.start(); + if (reportProcess.waitFor() != 0) { + throw new Exception("Failed to generate Allure report"); + } + + List zipCommand = Arrays.asList( + "zip", "-qr", "allure.zip", "target" + ); + ProcessBuilder zipBuilder = new ProcessBuilder(zipCommand); + zipBuilder.inheritIO(); + Process zipProcess = zipBuilder.start(); + if (zipProcess.waitFor() != 0) { + throw new Exception("Failed to create zip file"); + } + + String s3BasePath = String.format("s3://%s/%s/%s", + System.getenv("resultsTargetBucket"), + System.getenv("resultsTargetBucketPath"), + System.getenv("buildId") + ); + + List uploadSiteCommand = Arrays.asList( + "aws", "s3", "cp", "target/site", s3BasePath + "/site/", "--recursive" + ); + ProcessBuilder uploadSiteBuilder = new ProcessBuilder(uploadSiteCommand); + uploadSiteBuilder.inheritIO(); + Process uploadSiteProcess = uploadSiteBuilder.start(); + if (uploadSiteProcess.waitFor() != 0) { + throw new Exception("Failed to upload site to S3"); + } + + List uploadZipCommand = Arrays.asList( + "aws", "s3", "cp", "allure.zip", s3BasePath + "/" + ); + ProcessBuilder uploadZipBuilder = new ProcessBuilder(uploadZipCommand); + uploadZipBuilder.inheritIO(); + Process uploadZipProcess = uploadZipBuilder.start(); + if (uploadZipProcess.waitFor() != 0) { + throw new Exception("Failed to upload zip to S3"); + } + } +} \ No newline at end of file From 075d8e7e70370ccf56a3420af7b0b67c3cc12fa2 Mon Sep 17 00:00:00 2001 From: Stefan Ranoszek Date: Mon, 9 Dec 2024 14:02:01 +0000 Subject: [PATCH 2/6] chore: compile --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index d68e9eb65..b2485ceb9 100644 --- a/pom.xml +++ b/pom.xml @@ -82,17 +82,17 @@ io.cucumber cucumber-java - test + compile io.cucumber cucumber-junit-platform-engine - test + compile org.junit.platform junit-platform-suite - test + compile @@ -152,13 +152,13 @@ io.cucumber cucumber-picocontainer - test + compile io.qameta.allure allure-cucumber7-jvm ${allure-cucumber7-jvm.version} - test + compile io.qameta.allure From b544e129ee6ea5f5578d55841c4740410e58c91e Mon Sep 17 00:00:00 2001 From: Stefan Ranoszek Date: Mon, 9 Dec 2024 14:11:58 +0000 Subject: [PATCH 3/6] chore: compile --- pom.xml | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index b2485ceb9..8e82f352f 100644 --- a/pom.xml +++ b/pom.xml @@ -41,24 +41,24 @@ 14400 - - - - org.junit - junit-bom - 5.10.1 - pom - import - - - io.cucumber - cucumber-bom - 7.15.0 - pom - import - - - + + + + + + + + + + + + + + + + + + @@ -82,17 +82,20 @@ io.cucumber cucumber-java - compile + test + io.cucumber cucumber-junit-platform-engine - compile + test + 7.20.1 org.junit.platform junit-platform-suite - compile + test + 1.11.3 @@ -152,13 +155,14 @@ io.cucumber cucumber-picocontainer - compile + test + 7.20.1 io.qameta.allure allure-cucumber7-jvm ${allure-cucumber7-jvm.version} - compile + test io.qameta.allure From e772f20a0809eed5a4ddee464c74fe89cb96215d Mon Sep 17 00:00:00 2001 From: Stefan Ranoszek Date: Mon, 9 Dec 2024 14:46:35 +0000 Subject: [PATCH 4/6] chore: conflicts --- pom.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 8e82f352f..63b2797c1 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 2.11.0 - 2.20.1 + 2.29.1 2.25.0 2.22.0 @@ -82,20 +82,21 @@ io.cucumber cucumber-java + 7.20.1 test io.cucumber cucumber-junit-platform-engine - test 7.20.1 + test org.junit.platform junit-platform-suite - test 1.11.3 + test @@ -155,8 +156,9 @@ io.cucumber cucumber-picocontainer - test 7.20.1 + test + io.qameta.allure From fd541ac97590c97d54715d84e0a04f2db1a4a72f Mon Sep 17 00:00:00 2001 From: Stefan Ranoszek Date: Mon, 9 Dec 2024 14:52:35 +0000 Subject: [PATCH 5/6] chore: conflicts --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 63b2797c1..7e40271d9 100644 --- a/pom.xml +++ b/pom.xml @@ -28,10 +28,10 @@ 1.3 1.9.21 - 3.2.0 + 3.9.9 3.2.5 3.2.5 - 3.2.4 + 3.6.0 1.10.0 1.17.1 2.10.1 From 2e6e44da395822937ba96aee869b3c5306cadfbf Mon Sep 17 00:00:00 2001 From: Stefan Ranoszek Date: Mon, 9 Dec 2024 15:00:27 +0000 Subject: [PATCH 6/6] chore: mvn plugin --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 7e40271d9..e76fa0f35 100644 --- a/pom.xml +++ b/pom.xml @@ -160,6 +160,11 @@ test + + org.apache.maven.plugins + maven-clean-plugin + 4.0.0-beta-1 + io.qameta.allure allure-cucumber7-jvm