From df7937b7bed0c308e4d6b6d97f6e2ed7406a5944 Mon Sep 17 00:00:00 2001 From: hechmi-dammak-xenit <107396375+hechmi-dammak-xenit@users.noreply.github.com> Date: Tue, 28 Nov 2023 15:37:21 +0100 Subject: [PATCH] DOCKER-442 fix solr backup numberToLive * DOCKER-442 fix solr backup numberToLive * DOCKER-442 add time to live integration test --- renovate.json => .github/renovate.json | 0 .github/workflows/ci.yml | 41 +++-- CHANGELOG.md | 8 + README.md | 42 ++++- build.gradle | 11 -- gradle.properties | 8 + integration-tests/build.gradle | 102 +++++------ integration-tests/solr6/overload.gradle | 7 - .../xenit/solr/backup/s3/SolrBackupTest.java | 158 +++++++++++------- .../test/resources/compose/docker-compose.yml | 6 +- integration-tests/src/test/resources/solr.xml | 4 +- settings.gradle | 3 - solr-backup/build.gradle | 23 ++- .../solr/backup/s3/S3BackupRepository.java | 48 ++++-- .../backup/s3/S3BackupRepositoryConfig.java | 6 +- .../xenit/solr/backup/s3/S3StorageClient.java | 14 +- .../backup/s3/S3BackupRepositoryTest.java | 54 ++++++ 17 files changed, 341 insertions(+), 194 deletions(-) rename renovate.json => .github/renovate.json (100%) create mode 100644 gradle.properties delete mode 100644 integration-tests/solr6/overload.gradle create mode 100644 solr-backup/src/test/java/eu/xenit/solr/backup/s3/S3BackupRepositoryTest.java diff --git a/renovate.json b/.github/renovate.json similarity index 100% rename from renovate.json rename to .github/renovate.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdcdb60..2827f80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,10 +3,11 @@ on: push: workflow_dispatch: env: - ORG_GRADLE_PROJECT_alfresco_nexus_username: ${{ secrets.ALFRESCO_NEXUS_USERNAME }} - ORG_GRADLE_PROJECT_alfresco_nexus_password: ${{ secrets.ALFRESCO_NEXUS_PASSWORD }} + GRADLE_OPTS: >- + -Dorg.gradle.project.org.alfresco.maven.nexus.username=${{ secrets.ALFRESCO_NEXUS_USERNAME }} + -Dorg.gradle.project.org.alfresco.maven.nexus.password=${{ secrets.ALFRESCO_NEXUS_PASSWORD }} jobs: - integration-tests: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -15,15 +16,11 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 11 - - name: Login to Docker - run: | - echo "${{ secrets.CLOUDSMITH_APIKEY }}" | docker login private.docker.xenit.eu --username "${{ secrets.CLOUDSMITH_USER }}" --password-stdin - - name: Test + - name: Integration test uses: gradle/gradle-build-action@v2 with: cache-read-only: false - arguments: | - integration-tests:solr6:integrationTest + arguments: test -x :integration-tests:test - name: Upload Test Artifact if: success() || failure() uses: actions/upload-artifact@v3 @@ -31,8 +28,29 @@ jobs: name: test-result path: /home/runner/work/**/build/reports retention-days: 2 + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Integration test + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: false + arguments: integration-tests:test + - name: Upload Test Artifact + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: integration-test-result + path: /home/runner/work/**/build/reports + retention-days: 2 publish: - needs: [ integration-tests ] + needs: [ test, integration-test ] runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/tags/v') }} steps: @@ -53,5 +71,4 @@ jobs: ORG_GRADLE_PROJECT_sonatype_password: ${{ secrets.SONATYPE_S01_PASSWORD }} with: cache-read-only: false - arguments: | - publish -PsigningKeyId=CDE3528F -i + arguments: publish -PsigningKeyId=CDE3528F -i diff --git a/CHANGELOG.md b/CHANGELOG.md index a935f21..8b49230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ title: Changelog - Solr Backup # Alfresco Backup Changelog +## v0.0.9 - 28-11-2023 + +* DOCKER-442 fix solr backup numberToLive + +## v0.0.8 - 20-11-2023 + +* DOCKER-441 improve solr backup documentation and add new env + ## v0.0.5 - 3-11-2023 * OUPDAUNTLE-54 drop aws keys and use integrated env variables diff --git a/README.md b/README.md index 39973ad..c22c888 100644 --- a/README.md +++ b/README.md @@ -25,20 +25,44 @@ Integration tests follow the same line: * trigger a restore /solr/alfresco/replication?command=restore&repository=s3&location=s3:/// * check if the restore was successful in a certain timeout (3 minutes) by following the output of /solr/alfresco/replication?command=restorestatus +## Setup + +you need to put the solr.xml file under /opt/alfresco-search-services/solrhome/ +``` + + + ${adminHandler:org.alfresco.solr.AlfrescoCoreAdminHandler} + + + ${S3_BUCKET_NAME:} + ${S3_ENDPOINT:http://s3.eu-central-1.amazonaws.com} + ${S3_REGION:eu-central-1} + ${S3_ACCESS_KEY:} + ${S3_SECRET_KEY:} + ${S3_PROXY_HOST:} + ${S3_PROXY_PORT:0} + ${S3_PATH_STYLE_ACCESS_ENABLED:false} + + + +``` + +and specify the Environment or Java variables ## Variables all of these variable can be set as environment variable or as a system property so that it is substituted in solr.xml -| Environment variable | Java system property | Default | required | -|----------------------|----------------------|--------------------------------------|----------| -| S3_ENDPOINT | -DS3_ENDPOINT | http://s3.eu-central-1.amazonaws.com | false | -| S3_BUCKET_NAME | -DS3_BUCKET_NAME | | true | -| S3_REGION | -DS3_REGION | eu-central-1 | false | -| S3_ACCESS_KEY | -DS3_ACCESS_KEY | | false | -| S3_SECRET_KEY | -DS3_SECRET_KEY | | false | -| S3_PROXY_HOST | -DS3_PROXY_HOST | | false | -| S3_PROXY_PORT | -DS3_PROXY_PORT | | false | +| Environment variable | Java system property | Default | required | +|------------------------------|--------------------------------|--------------------------------------|----------| +| S3_ENDPOINT | -DS3_ENDPOINT | http://s3.eu-central-1.amazonaws.com | false | +| S3_BUCKET_NAME | -DS3_BUCKET_NAME | | true | +| S3_REGION | -DS3_REGION | eu-central-1 | false | +| S3_ACCESS_KEY | -DS3_ACCESS_KEY | | false | +| S3_SECRET_KEY | -DS3_SECRET_KEY | | false | +| S3_PROXY_HOST | -DS3_PROXY_HOST | | false | +| S3_PROXY_PORT | -DS3_PROXY_PORT | | false | +| S3_PATH_STYLE_ACCESS_ENABLED | -DS3_PATH_STYLE_ACCESS_ENABLED | false | false | ## Testing against DataCore Swarm docker diff --git a/build.gradle b/build.gradle index a8eff1c..67620d9 100644 --- a/build.gradle +++ b/build.gradle @@ -3,22 +3,11 @@ plugins { id 'eu.xenit.docker-compose' version '5.4.0' apply false } -def copyPropertyValueIfExists(sourcePropertyName, targetPropertyName) { - if (project.hasProperty(sourcePropertyName)) { - project.ext[targetPropertyName] = project.property(sourcePropertyName) - } -} - subprojects { apply plugin: 'java' def baseVersion = System.getenv("TAG_VERSION") ?: 'v0.0.6' version = baseVersion[1..baseVersion.length() - 1] - // It is not possible to set properties with a dot via GitHub Actions env variables, therefore we introduce support - // for a non-dotted-equivalent - copyPropertyValueIfExists('alfresco_nexus_username', 'org.alfresco.maven.nexus.username') - copyPropertyValueIfExists('alfresco_nexus_password', 'org.alfresco.maven.nexus.password') - repositories { mavenCentral() maven { diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..09ca40e --- /dev/null +++ b/gradle.properties @@ -0,0 +1,8 @@ +junitJupiterVersion=5.4.2 +mockitoVersion=2.27.0 +solrVersion=6.6.5 +assVersion=2.0.8.2 +amazonVersion=1.12.32 +jaxBVersion=2.3.2 +restAssuredVersion=4.0.0 +awaitablityVersion=4.2.0 \ No newline at end of file diff --git a/integration-tests/build.gradle b/integration-tests/build.gradle index 6d95c16..c0c400c 100644 --- a/integration-tests/build.gradle +++ b/integration-tests/build.gradle @@ -1,74 +1,60 @@ -plugins { - id 'java' - id 'idea' +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'eu.xenit.docker' +apply plugin: 'eu.xenit.docker-compose.auto' + +ext { + solrVersion = '6.6.5' + assVersion = '2.0.6' + solrBaseImage = 'docker.io/xenit/alfresco-solr6-xenit:2.0.6' + alfrescoimage = 'docker.io/xenit/alfresco-repository-community:7.3.0' + flavor = 'solr6' } -dependencies { - testImplementation group: 'io.rest-assured', name: 'rest-assured', version: '3.0.1' - testImplementation group: 'io.rest-assured', name: 'json-path', version: '3.0.1' - testImplementation group: 'io.rest-assured', name: 'rest-assured-common', version: '3.0.1' - testImplementation "org.junit.jupiter:junit-jupiter-engine:5.4.2" - testImplementation "org.junit.jupiter:junit-jupiter-params:5.4.2" - testRuntimeOnly group: 'org.glassfish.jaxb', name: 'jaxb-runtime', version: '2.3.2' -} +description = "Solr ${flavor} with backup" -test { - enabled = false - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } +configurations { + backupJar } -subprojects { - apply plugin: 'java' - apply plugin: 'eu.xenit.docker' - apply plugin: 'eu.xenit.docker-compose.auto' - apply from: "${project.projectDir}/overload.gradle" - - description = "Solr ${flavor} with backup" - - configurations { - amazonSdkCore - amazonSdkS3 - backupJar - } - - dependencies { - amazonSdkCore group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.32', ext: 'jar' - amazonSdkS3 group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.32', ext: 'jar' +dependencies { + backupJar project(path: ":solr-backup") - backupJar project(path: ":solr-backup") - } + testImplementation "io.rest-assured:rest-assured:${restAssuredVersion}" + testImplementation "io.rest-assured:json-path:${restAssuredVersion}" + testImplementation "io.rest-assured:rest-assured-common:${restAssuredVersion}" + testImplementation "org.junit.jupiter:junit-jupiter-engine:${junitJupiterVersion}" + testImplementation "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}" + testImplementation "org.awaitility:awaitility:${awaitablityVersion}" + testImplementation platform("com.amazonaws:aws-java-sdk-bom:${amazonVersion}") - task integrationTest(type: Test, group: "verification") { - useJUnitPlatform() - testClassesDirs = project.parent.sourceSets.test.output.classesDirs - classpath = project.parent.sourceSets.test.runtimeClasspath - outputs.upToDateWhen { false } - } + testImplementation('com.amazonaws:aws-java-sdk-core') + testImplementation('com.amazonaws:aws-java-sdk-s3') + testImplementation("com.amazonaws:aws-java-sdk-sts") + testRuntimeOnly "org.glassfish.jaxb:jaxb-runtime:${jaxBVersion}" +} +test { + useJUnitPlatform() +} - createDockerFile { - from "${solrBaseImage}" +createDockerFile { + from "${solrBaseImage}" - dependsOn(configurations.backupJar) + dependsOn(configurations.backupJar) - if (flavor == "solr6") { - smartCopy "${project.parent.projectDir}/src/test/resources/solr.xml", "/opt/alfresco-search-services/solrhome/solr.xml" - smartCopy configurations.backupJar, "/opt/alfresco-search-services/solrhome/lib/" - smartCopy configurations.amazonSdkCore.singleFile, "/opt/alfresco-search-services/solrhome/lib/" - smartCopy configurations.amazonSdkS3.singleFile, "/opt/alfresco-search-services/solrhome/lib/" - } + if (flavor == "solr6") { + smartCopy "${project.projectDir}/src/test/resources/solr.xml", "/opt/alfresco-search-services/solrhome/solr.xml" + smartCopy configurations.backupJar, "/opt/alfresco-search-services/solrhome/lib/" } +} - dockerCompose { - environment.put 'ALFRESCO_IMAGE', project.alfrescoimage - useComposeFiles = [ - "${project.parent.projectDir}/src/test/resources/compose/docker-compose.yml" - ] - isRequiredBy(project.tasks.integrationTest) - } +dockerCompose { + environment.put 'ALFRESCO_IMAGE', project.alfrescoimage + useComposeFiles = [ + "${project.projectDir}/src/test/resources/compose/docker-compose.yml" + ] + isRequiredBy(project.tasks.test) } diff --git a/integration-tests/solr6/overload.gradle b/integration-tests/solr6/overload.gradle deleted file mode 100644 index 88432d3..0000000 --- a/integration-tests/solr6/overload.gradle +++ /dev/null @@ -1,7 +0,0 @@ -ext { - solrVersion = '6.6.5' - assVersion = '2.0.6' - solrBaseImage = 'private.docker.xenit.eu/alfresco-enterprise/alfresco-solr6:2.0.6' - alfrescoimage ='docker.io/xenit/alfresco-repository-community:7.3.0' - flavor = 'solr6' -} diff --git a/integration-tests/src/test/java/eu/xenit/solr/backup/s3/SolrBackupTest.java b/integration-tests/src/test/java/eu/xenit/solr/backup/s3/SolrBackupTest.java index 184efcb..12a0c2b 100644 --- a/integration-tests/src/test/java/eu/xenit/solr/backup/s3/SolrBackupTest.java +++ b/integration-tests/src/test/java/eu/xenit/solr/backup/s3/SolrBackupTest.java @@ -1,5 +1,13 @@ package eu.xenit.solr.backup.s3; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ObjectListing; import groovy.util.logging.Slf4j; import io.restassured.RestAssured; import io.restassured.builder.RequestSpecBuilder; @@ -8,14 +16,20 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.util.concurrent.TimeUnit; import static io.restassured.RestAssured.given; import static java.lang.Thread.sleep; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) class SolrBackupTest { private static final Log log = LogFactory.getLog(SolrBackupTest.class); static RequestSpecification spec; @@ -23,12 +37,18 @@ class SolrBackupTest { static RequestSpecification specBackupDetails; static RequestSpecification specRestore; static RequestSpecification specRestoreStatus; + static AmazonS3 s3Client; + static final String BUCKET = "bucket"; @BeforeEach public void setup() { String basePathSolr = "solr/alfresco"; String basePathSolrBackup = "solr/alfresco/replication"; String solrHost = System.getProperty("solr.host", "localhost"); + s3Client = createInternalClient("us-east-1", + "http://localhost:4566", + "access_key", + "9access_key"); int solrPort = 0; try { solrPort = Integer.parseInt(System.getProperty("solr.tcp.8080", "8080")); @@ -52,8 +72,7 @@ public void setup() { .setBasePath(basePathSolrBackup) .addParam("command", "backup") .addParam("repository", "s3") - .addParam("location", "s3:///") - .addParam("numberToKeep", "3") + .addParam("numberToKeep", "2") .addParam("wt", "json") .build(); specBackupDetails = new RequestSpecBuilder() @@ -69,7 +88,6 @@ public void setup() { .setBasePath(basePathSolrBackup) .addParam("command", "restore") .addParam("repository", "s3") - .addParam("location", "s3:///") .build(); specRestoreStatus = new RequestSpecBuilder() .setBaseUri(baseURISolr) @@ -88,8 +106,61 @@ public void setup() { } } + + @Test + @Order(2) + void testRestoreEndpoint() { + given() + .spec(specRestore) + .when() + .get() + .then() + .statusCode(200); + System.out.println("Restore triggered, will wait maximum 3 minutes"); + long startTime = System.currentTimeMillis(); + await().atMost(180, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS).until(() -> { + String status = given() + .spec(specRestoreStatus) + .when() + .get() + .then() + .statusCode(200) + .extract() + .path("restorestatus.status"); + System.out.println("elapsed = " + (System.currentTimeMillis() - startTime) + "with status= " + status); + return "success".equals(status); + }); + } + @Test - void testBackupEndpoint() { + @Order(1) + void testBackupWithNumberToLiveEndpoint() { + validateSnapshotCount(0); + callBackupEndpoint(1); + validateSnapshotCount(1); + callBackupEndpoint(2); + validateSnapshotCount(2); + callBackupEndpoint(3); + validateSnapshotCount(2); + } + + + void validateSnapshotCount(long count) { + ObjectListing objectListing = s3Client.listObjects(BUCKET); + await().atMost(180, TimeUnit.SECONDS) + .until(() -> objectListing + .getObjectSummaries() + .stream() + .filter(s3ObjectSummary -> s3ObjectSummary.getSize() == 0 + && s3ObjectSummary.getKey().contains("snapshot")) + .count() == count); + + } + private void callBackupEndpoint() { + callBackupEndpoint(0); + } + private void callBackupEndpoint(int count) { String status = given() .spec(specBackup) .when() @@ -99,60 +170,33 @@ void testBackupEndpoint() { .extract() .path("status"); assertEquals("OK", status); - System.out.println("Backup triggered, will wait maximum 6 minutes"); - Object backup = null; - long timeout = 500000; - long elapsed = 0; - while (backup == null && elapsed < timeout) { - backup = given() - .spec(specBackupDetails) - .when() - .get() - .then() - .statusCode(200) - .extract() - .path("details.backup"); - System.out.println("elapsed =" + elapsed); - try { - sleep(1000); - elapsed += 1000; - } catch (InterruptedException e) { - log.error(e); - } - } - assertTrue(elapsed < timeout); + System.out.println("Backup triggered" + (count == 0 ? "" : count + " time ") + ", will wait maximum 9 minutes"); + long startTime = System.currentTimeMillis(); + await().atMost(540, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> { + Object backup = given() + .spec(specBackupDetails) + .when() + .get() + .then() + .statusCode(200) + .extract() + .path("details.backup"); + System.out.println("elapsed = " + (System.currentTimeMillis() - startTime)); + return backup != null; + }); } - @Test - void testRestoreEndpoint() { - given() - .spec(specRestore) - .when() - .get() - .then() - .statusCode(200); - System.out.println("Restore triggered, will wait maximum 3 minutes"); - String status = ""; - long timeout = 180000; - long elapsed = 0; - while (!"success".equals(status) && elapsed < timeout) { - status = given() - .spec(specRestoreStatus) - .when() - .get() - .then() - .statusCode(200) - .extract() - .path("restorestatus.status"); - System.out.println("status=" + status); - try { - sleep(1000); - elapsed += 1000; - } catch (InterruptedException e) { - log.error(e); - } - } - assertTrue(elapsed < timeout); + private AmazonS3 createInternalClient( + String region, String endpoint, String accessKey, String secretKey) { + ClientConfiguration clientConfig = new ClientConfiguration().withProtocol(Protocol.HTTPS); + AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfig); + clientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))); + clientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)); + clientBuilder.withPathStyleAccessEnabled(true); + return clientBuilder.build(); } + } diff --git a/integration-tests/src/test/resources/compose/docker-compose.yml b/integration-tests/src/test/resources/compose/docker-compose.yml index d3a648d..3924f47 100644 --- a/integration-tests/src/test/resources/compose/docker-compose.yml +++ b/integration-tests/src/test/resources/compose/docker-compose.yml @@ -33,18 +33,18 @@ services: ports: - "8080:8080" - "8000:8000" - - "5000:5000" environment: - ALFRESCO_SSL=secret - DEBUG=true - JMX_ENABLED=true - JAVA_XMX=1024M - - JAVA_OPTS_SSL=-Dalfresco.secureComms.secret=mysolrsecret + - ALFRESCO_SECRET=mysolrsecret - S3_ENDPOINT=http://localstack:4566 - S3_REGION=us-east-1 - S3_BUCKET_NAME=bucket - S3_ACCESS_KEY=access_key - - S3_SECRET_KEY=secret_key + - S3_SECRET_KEY=access_key + - S3_PATH_STYLE_ACCESS_ENABLED=true localstack: container_name: localstack diff --git a/integration-tests/src/test/resources/solr.xml b/integration-tests/src/test/resources/solr.xml index 9b050b6..3894bd6 100644 --- a/integration-tests/src/test/resources/solr.xml +++ b/integration-tests/src/test/resources/solr.xml @@ -1,9 +1,8 @@ ${adminHandler:org.alfresco.solr.AlfrescoCoreAdminHandler} - - + ${S3_BUCKET_NAME:} ${S3_ENDPOINT:http://s3.eu-central-1.amazonaws.com} ${S3_REGION:eu-central-1} @@ -11,6 +10,7 @@ ${S3_SECRET_KEY:} ${S3_PROXY_HOST:} ${S3_PROXY_PORT:0} + ${S3_PATH_STYLE_ACCESS_ENABLED:false} diff --git a/settings.gradle b/settings.gradle index a36533f..fe0c07e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -3,6 +3,3 @@ rootProject.name = 'solr-backup' include ':solr-backup' include ':integration-tests' -["solr6"].each { version -> - include ":integration-tests:${version}" -} diff --git a/solr-backup/build.gradle b/solr-backup/build.gradle index ffe8ef4..46d8c54 100644 --- a/solr-backup/build.gradle +++ b/solr-backup/build.gradle @@ -5,11 +5,6 @@ plugins { description = "Xenit backup" group = 'eu.xenit.solr-backup' -ext { - solrVersion = '6.6.5' - assVersion = '2.0.6' -} - java { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 @@ -25,12 +20,24 @@ dependencies { } compileOnly "org.alfresco:alfresco-search:${assVersion}" - implementation platform('com.amazonaws:aws-java-sdk-bom:1.12.32') + implementation platform("com.amazonaws:aws-java-sdk-bom:${amazonVersion}") - implementation ('com.amazonaws:aws-java-sdk-core') - implementation ('com.amazonaws:aws-java-sdk-s3') + implementation('com.amazonaws:aws-java-sdk-core') + implementation('com.amazonaws:aws-java-sdk-s3') implementation("com.amazonaws:aws-java-sdk-sts") + testImplementation("org.apache.solr:solr-core:${solrVersion}") { + exclude group: 'org.restlet.jee' // Only available in JCenter, not essential in this project. + } + testImplementation "org.alfresco:alfresco-search:${assVersion}" + + testImplementation "org.junit.jupiter:junit-jupiter-engine:${junitJupiterVersion}" + testImplementation "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}" + testImplementation "org.mockito:mockito-core:${mockitoVersion}" + testImplementation "org.mockito:mockito-junit-jupiter:${mockitoVersion}" } +test { + useJUnitPlatform() +} apply from: "${rootProject.projectDir}/publish.gradle" \ No newline at end of file diff --git a/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepository.java b/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepository.java index 6d66979..9b60bf7 100644 --- a/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepository.java +++ b/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepository.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; @@ -55,6 +54,11 @@ public class S3BackupRepository implements BackupRepository { static final String S3_SCHEME = "s3"; private NamedList config; + + public void setClient(S3StorageClient client) { + this.client = client; + } + private S3StorageClient client; @Override @@ -76,26 +80,34 @@ public T getConfigProperty(String name) { return (T) this.config.get(name); } - @Override - public URI createURI(String location) { - if (StringUtils.isEmpty(location)) { - throw new IllegalArgumentException("cannot create URI with an empty location"); + @Override + public URI createURI(String location) { + if (StringUtils.isEmpty(location)) { + throw new IllegalArgumentException("cannot create URI with an empty location"); + } + + try { + URI result = getUri(location); + createBackUpDirectory(result); + return result; + } catch (URISyntaxException ex) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ex); + } } - URI result; - try { - if (location.startsWith(S3_SCHEME + ":")) { - result = new URI(location); - } else if (location.startsWith("/")) { - result = new URI(S3_SCHEME, null, location, null); - } else { - result = new URI(S3_SCHEME, null, "/" + location, null); - } - createBackUpDirectory(result); - return result; - } catch (URISyntaxException ex) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ex); + private URI getUri(String location) throws URISyntaxException { + StringBuilder locationResult = new StringBuilder(); + if (!location.startsWith("/") && !location.startsWith(S3_SCHEME + ":")) { + locationResult.append("/"); + } + locationResult.append(location); + if (!location.endsWith("/")) { + locationResult.append("/"); + } + if (location.startsWith(S3_SCHEME + ":")) { + return new URI(locationResult.toString()); } + return new URI(S3_SCHEME, null, locationResult.toString(), null); } private void createBackUpDirectory(URI result) { diff --git a/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepositoryConfig.java b/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepositoryConfig.java index aec834a..5cb0428 100644 --- a/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepositoryConfig.java +++ b/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3BackupRepositoryConfig.java @@ -31,6 +31,7 @@ public class S3BackupRepositoryConfig { public static final String S3_ENDPOINT = "s3.endpoint"; public static final String S3_PROXY_HOST = "s3.proxy.host"; public static final String S3_PROXY_PORT = "s3.proxy.port"; + public static final String S3_PATH_STYLE_ACCESS_ENABLED = "s3.path.style.access.enabled"; private final String bucketName; private final String region; @@ -39,6 +40,8 @@ public class S3BackupRepositoryConfig { private final String proxyHost; private final int proxyPort; private final String endpoint; + private final Boolean pathStyleAccessEnabled; + public S3BackupRepositoryConfig(NamedList config) { region = getStringConfig(config, S3_REGION); @@ -48,13 +51,14 @@ public S3BackupRepositoryConfig(NamedList config) { endpoint = getStringConfig(config, S3_ENDPOINT); accessKey = getStringConfig(config, S3_ACCESS_KEY); secretKey = getStringConfig(config, S3_SECRET_KEY); + pathStyleAccessEnabled = getBooleanConfig(config, S3_PATH_STYLE_ACCESS_ENABLED); } /** * @return a {@link S3StorageClient} from the provided config. */ public S3StorageClient buildClient() { - return new S3StorageClient(bucketName, region, proxyHost, proxyPort, endpoint, accessKey, secretKey); + return new S3StorageClient(bucketName, region, proxyHost, proxyPort, endpoint, accessKey, secretKey, pathStyleAccessEnabled); } private static String getStringConfig(NamedList config, String property) { diff --git a/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3StorageClient.java b/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3StorageClient.java index b5a25be..b719c4d 100644 --- a/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3StorageClient.java +++ b/solr-backup/src/main/java/eu/xenit/solr/backup/s3/S3StorageClient.java @@ -85,8 +85,8 @@ class S3StorageClient { private final String bucketName; S3StorageClient( - String bucketName, String region, String proxyHost, int proxyPort, String endpoint, String accessKey, String secretKey) { - this(createInternalClient(region, proxyHost, proxyPort, endpoint, accessKey, secretKey), bucketName); + String bucketName, String region, String proxyHost, int proxyPort, String endpoint, String accessKey, String secretKey, Boolean pathStyleAccessEnabled) { + this(createInternalClient(region, proxyHost, proxyPort, endpoint, accessKey, secretKey, pathStyleAccessEnabled), bucketName); } @VisibleForTesting @@ -96,7 +96,12 @@ class S3StorageClient { } private static AmazonS3 createInternalClient( - String region, String proxyHost, int proxyPort, String endpoint, String accessKey, String secretKey) { + String region, + String proxyHost, + int proxyPort, + String endpoint, + String accessKey, + String secretKey, Boolean pathStyleAccessEnabled) { ClientConfiguration clientConfig = new ClientConfiguration().withProtocol(Protocol.HTTPS); // If configured, add proxy if (!StringUtils.isEmpty(proxyHost)) { @@ -111,7 +116,6 @@ private static AmazonS3 createInternalClient( */ AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard() - .enablePathStyleAccess() .withClientConfiguration(clientConfig); if (!(StringUtils.isEmpty(accessKey) || StringUtils.isEmpty(secretKey))) { clientBuilder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))); @@ -124,7 +128,7 @@ private static AmazonS3 createInternalClient( } else { clientBuilder.setRegion(region); } - + clientBuilder.withPathStyleAccessEnabled(pathStyleAccessEnabled); return clientBuilder.build(); } diff --git a/solr-backup/src/test/java/eu/xenit/solr/backup/s3/S3BackupRepositoryTest.java b/solr-backup/src/test/java/eu/xenit/solr/backup/s3/S3BackupRepositoryTest.java new file mode 100644 index 0000000..c682159 --- /dev/null +++ b/solr-backup/src/test/java/eu/xenit/solr/backup/s3/S3BackupRepositoryTest.java @@ -0,0 +1,54 @@ +package eu.xenit.solr.backup.s3; + + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URI; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@ExtendWith(MockitoExtension.class) +class S3BackupRepositoryTest { + S3BackupRepository s3BackupRepository; + @Mock + S3StorageClient client; + + @BeforeEach + public void setup() { + s3BackupRepository = new S3BackupRepository(); + s3BackupRepository.setClient(client); + } + + @ParameterizedTest + @CsvSource({ + "example/path, /example/path/", + "/another/path/, /another/path/", + "s3://bucket/object-key, /object-key/" + }) + void testCreateURIWithDifferentPaths(String location, String expectedPath) throws S3Exception { + Mockito.when(client.pathExists(Mockito.any(String.class))).thenReturn(true); + + URI result = s3BackupRepository.createURI(location); + + assertNotNull(result); + assertEquals("s3", result.getScheme()); + assertEquals(expectedPath, result.getPath()); + } + + + @Test + void testCreateURIWithEmptyLocation() { + assertThrows(IllegalArgumentException.class, + () -> s3BackupRepository.createURI("")); + + } +} \ No newline at end of file