GH-1821 Support StoragePolicy (Resolve #1821)

dzikoysk · Aug 25, 2023 · 5db56cf · 5db56cf
1 parent dc03731
commit 5db56cf
Show file tree

Hide file tree

Showing 8 changed files with 120 additions and 47 deletions.
diff --git a/reposilite-backend/src/integration/kotlin/com/reposilite/maven/MavenIntegrationTest.kt b/reposilite-backend/src/integration/kotlin/com/reposilite/maven/MavenIntegrationTest.kt
@@ -31,7 +31,6 @@ import kong.unirest.Unirest.delete
 import kong.unirest.Unirest.get
 import kong.unirest.Unirest.head
 import kong.unirest.Unirest.put
-import kotlinx.coroutines.runBlocking
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
@@ -166,34 +165,4 @@ internal abstract class MavenIntegrationTest : MavenIntegrationSpecification() {
         assertThat(response.body).contains("Reposilite - 404 Not Found")
     }
 
-    @Test
-    fun `should proxy remote file`() = runBlocking {
-        // given: a remote server and artifact
-        useProxiedHost("releases", "com/reposilite/remote.jar", "content") { gav, content ->
-            // when: non-existing file is requested
-            val notFoundResponse = get("$base/proxied/not/found.jar").asString()
-
-            // then: service responds with 404 status page
-            assertThat(notFoundResponse.isSuccess).isFalse
-
-            // when: file that exists in remote repository is requested
-            val response = get("$base/proxied/$gav").asString()
-
-            // then: service responds with its content
-            assertThat(response.body).isEqualTo(content)
-            assertThat(response.isSuccess).isTrue
-        }
-    }
-
-    @Test
-    fun `should not proxy file with forbidden extension`() = runBlocking {
-        // given: a remote server and artifact
-        useProxiedHost("releases", "com/reposilite/remote.file", "content") { gav, _ ->
-            // when: file that exists in remote repository is requested
-            val response = get("$base/proxied/$gav").asString()
-            // then: service responds with 404 status page as .file extension is not allowed
-            assertThat(response.isSuccess).isFalse
-        }
-    }
-
 }
diff --git a/...silite-backend/src/integration/kotlin/com/reposilite/maven/MavenMirrorsIntegrationTest.kt b/...silite-backend/src/integration/kotlin/com/reposilite/maven/MavenMirrorsIntegrationTest.kt
@@ -0,0 +1,81 @@
+@file:Suppress("FunctionName")
+
+package com.reposilite.maven
+
+import com.reposilite.RecommendedLocalSpecificationJunitExtension
+import com.reposilite.RecommendedRemoteSpecificationJunitExtension
+import com.reposilite.maven.api.LookupRequest
+import com.reposilite.maven.specification.MavenIntegrationSpecification
+import com.reposilite.storage.api.toLocation
+import kong.unirest.Unirest
+import kotlinx.coroutines.runBlocking
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+
+@ExtendWith(RecommendedLocalSpecificationJunitExtension::class)
+internal class LocalMavenMirrorsIntegrationTest : MavenMirrorsIntegrationTest()
+
+@ExtendWith(RecommendedRemoteSpecificationJunitExtension::class)
+internal class RemoteMavenMirrorsIntegrationTest : MavenMirrorsIntegrationTest()
+
+internal abstract class MavenMirrorsIntegrationTest : MavenIntegrationSpecification() {
+
+    @Test
+    fun `should proxy remote file`() = runBlocking {
+        // given: a remote server and artifact
+        useProxiedHost("releases", "com/reposilite/remote.jar", "content") { gav, content ->
+            // when: non-existing file is requested
+            val notFoundResponse = Unirest.get("$base/proxied/not/found.jar").asString()
+
+            // then: service responds with 404 status page
+            assertThat(notFoundResponse.isSuccess).isFalse
+
+            // when: file that exists in remote repository is requested
+            val response = Unirest.get("$base/proxied/$gav").asString()
+
+            // then: service responds with its content
+            assertThat(response.body).isEqualTo(content)
+            assertThat(response.isSuccess).isTrue
+        }
+    }
+
+    @Test
+    fun `should not proxy file with forbidden extension`() = runBlocking {
+        // given: a remote server and artifact
+        useProxiedHost("releases", "com/reposilite/remote.file", "content") { gav, _ ->
+            // when: file that exists in remote repository is requested
+            val response = Unirest.get("$base/proxied/$gav").asString()
+            // then: service responds with 404 status page as .file extension is not allowed
+            assertThat(response.isSuccess).isFalse
+        }
+    }
+
+    @Test
+    fun `should prioritize upstream metadata file over local copy`() = runBlocking {
+        // given: a remote server and artifact
+        useProxiedHost("releases", "com/reposilite/maven-metadata.xml", "upstream") { gav, _ ->
+            // and: local repository with cached metadata file
+            useDocument("proxied", "com/reposilite", "maven-metadata.xml", "local", true)
+
+            // when: metadata file is requested
+            val response = Unirest.get("$base/proxied/$gav").asString()
+
+            // then: service responds with upstream metadata file
+            assertThat(response.body).isEqualTo("upstream")
+            assertThat(response.isSuccess).isTrue
+
+            // and: local metadata file is updated
+            val localFile = mavenFacade.findFile(
+                LookupRequest(
+                    accessToken = null,
+                    repository = "proxied",
+                    gav = "com/reposilite/maven-metadata.xml".toLocation(),
+                )
+            )
+            assertThat(localFile.isOk).isTrue
+            assertThat(localFile.get().second.readAllBytes().decodeToString()).isEqualTo("upstream")
+        }
+    }
+
+}
diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/MirrorHost.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/MirrorHost.kt
@@ -24,3 +24,8 @@ data class MirrorHost(
     val configuration: MirroredRepositorySettings,
     val client: RemoteClient
 )
+
+enum class StoragePolicy {
+    PRIORITIZE_UPSTREAM_METADATA,
+    STRICT
+}
diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/MirrorService.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/MirrorService.kt
@@ -18,6 +18,7 @@ package com.reposilite.maven
 
 import com.reposilite.journalist.Journalist
 import com.reposilite.journalist.Logger
+import com.reposilite.maven.api.METADATA_FILE
 import com.reposilite.maven.application.MirroredRepositorySettings
 import com.reposilite.shared.ErrorResponse
 import com.reposilite.shared.notFoundError
@@ -31,6 +32,9 @@ import java.io.InputStream
 
 internal class MirrorService(private val journalist: Journalist) : Journalist {
 
+    fun shouldPrioritizeMirrorRepository(repository: Repository, gav: Location): Boolean =
+        repository.storagePolicy == StoragePolicy.PRIORITIZE_UPSTREAM_METADATA && gav.getSimpleName().contains(METADATA_FILE)
+
     fun findRemoteDetails(repository: Repository, gav: Location): Result<out FileDetails, ErrorResponse> =
         searchInRemoteRepositories(repository, gav) { (host, config, client) ->
             client.head("${host.removeSuffix("/")}/$gav", config.authorization, config.connectTimeout, config.readTimeout)

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/Repository.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/Repository.kt
@@ -34,6 +34,7 @@ class Repository internal constructor(
     val preserveSnapshots: Boolean,
     val mirrorHosts: List<MirrorHost>,
     val storageProvider: StorageProvider,
+    val storagePolicy: StoragePolicy
 ) {
 
     init {

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryFactory.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryFactory.kt
@@ -46,12 +46,16 @@ internal class RepositoryFactory(
             redeployment = configuration.redeployment,
             preserveSnapshots = configuration.preserveSnapshots,
             mirrorHosts = configuration.proxied.mapNotNull { createMirroredHostConfiguration(it) },
-            storageProvider = storageFacade.createStorageProvider(
-                failureFacade = failureFacade,
-                workingDirectory = workingDirectory.resolve(repositoriesDirectory),
-                repository = repositoryName,
-                storageSettings = configuration.storageProvider
-            ) ?: throw IllegalArgumentException("Unknown storage provider '${configuration.storageProvider.type}'")
+            storageProvider =
+                storageFacade
+                    .createStorageProvider(
+                        failureFacade = failureFacade,
+                        workingDirectory = workingDirectory.resolve(repositoriesDirectory),
+                        repository = repositoryName,
+                        storageSettings = configuration.storageProvider
+                    )
+                    ?: throw IllegalArgumentException("Unknown storage provider '${configuration.storageProvider.type}'"),
+            storagePolicy = configuration.storagePolicy
         )
 
     private fun createMirroredHostConfiguration(configurationSource: MirroredRepositorySettings): MirrorHost? {

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryService.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryService.kt
@@ -134,17 +134,20 @@ internal class RepositoryService(
             .let { extensions.emitEvent(ResolvedFileEvent(accessToken, repository, gav, it)).result }
 
     private fun findInputStream(repository: Repository, gav: Location): Result<InputStream, ErrorResponse> =
-        if (repository.storageProvider.exists(gav)) {
-            logger.debug("Gav '$gav' found in '${repository.name}' repository")
-            repository.storageProvider.getFile(gav)
-        } else {
-            logger.debug("Cannot find '$gav' in '${repository.name}' repository, requesting proxied repositories")
-            mirrorService.findRemoteFile(repository, gav)
+        when {
+            !mirrorService.shouldPrioritizeMirrorRepository(repository, gav) && repository.storageProvider.exists(gav) -> { // todo: add fallback to local for shouldPrioritizeMirrorRepository
+                logger.debug("Gav '$gav' found in '${repository.name}' repository")
+                repository.storageProvider.getFile(gav)
+            }
+            else -> {
+                logger.debug("Cannot find '$gav' in '${repository.name}' repository, requesting proxied repositories")
+                mirrorService.findRemoteFile(repository, gav)
+            }
         }
 
     private fun findDetails(accessToken: AccessTokenIdentifier?, repository: Repository, gav: Location): Result<out FileDetails, ErrorResponse> =
         when {
-            repository.storageProvider.exists(gav) -> findLocalDetails(accessToken, repository, gav)
+            !mirrorService.shouldPrioritizeMirrorRepository(repository, gav) && repository.storageProvider.exists(gav) -> findLocalDetails(accessToken, repository, gav) // todo: add fallback to local for shouldPrioritizeMirrorRepository
             else -> findProxiedDetails(repository, gav)
         }.peek {
             recordResolvedRequest(Identifier(repository.name, gav.toString()), it)

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenSettings.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenSettings.kt
@@ -19,6 +19,7 @@ package com.reposilite.maven.application
 import com.reposilite.configuration.shared.api.Doc
 import com.reposilite.configuration.shared.api.Min
 import com.reposilite.configuration.shared.api.SharedSettings
+import com.reposilite.maven.StoragePolicy
 import com.reposilite.maven.RepositoryVisibility
 import com.reposilite.maven.RepositoryVisibility.PRIVATE
 import com.reposilite.maven.RepositoryVisibility.PUBLIC
@@ -48,13 +49,18 @@ data class RepositorySettings(
     val id: String = "",
     @get:Doc(title = "Visibility", description = "The visibility of this repository.")
     val visibility: RepositoryVisibility = PUBLIC,
-    @get:Doc(title = "Storage provider", description = "The storage type of this repository.")
-    @get:OneOf(FileSystemStorageProviderSettings::class, S3StorageProviderSettings::class)
-    val storageProvider: StorageProviderSettings = FileSystemStorageProviderSettings(),
     @get:Doc(title = "Redeployment", description = "Does this repository accept redeployment of the same artifact version.")
     val redeployment: Boolean = false,
     @get:Doc(title = "Preserved snapshots", "By default Reposilite deletes all deprecated build files. If you'd like to preserve them, set this property to true.")
     val preserveSnapshots: Boolean = false,
+    @get:Doc(title = "Storage provider", description = "The storage type of this repository.")
+    @get:OneOf(FileSystemStorageProviderSettings::class, S3StorageProviderSettings::class)
+    val storageProvider: StorageProviderSettings = FileSystemStorageProviderSettings(),
+    @get:Doc(title = "Storage policy", description = """
+        By default Reposilite prioritizes upstream metadata over a cached version, so it'll always try to fetch the latest version of the artifact from the remote repository. <br/>
+        If you'd like to go full offline mode, set this property to STRICT.
+    """)
+    val storagePolicy: StoragePolicy = StoragePolicy.PRIORITIZE_UPSTREAM_METADATA,
     @get:Doc(title = "Mirrored repositories", description = "List of mirrored repositories associated with this repository.")
     val proxied: List<MirroredRepositorySettings> = listOf()
 ) : SharedSettings