feat: 支持Generic Remote仓库下载 TencentBlueKing#1400 (TencentBlueKing#1406)

* feat: 支持Generic Remote仓库下载 TencentBlueKing#1400

* feat: 支持Generic Remote仓库使用平台账号认证 TencentBlueKing#1400

* feat: 支持搜索Generic Remote仓库 TencentBlueKing#1400

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt

@@ -35,7 +35,6 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders
 import com.tencent.bkrepo.common.artifact.api.ArtifactFile
 import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.NetworkProxyConfiguration
 import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration
-import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteCredentialsConfiguration
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
@@ -231,7 +230,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
         builder.proxy(createProxy(configuration.network.proxy))
         builder.proxyAuthenticator(createProxyAuthenticator(configuration.network.proxy))
         if (addInterceptor) {
-            createAuthenticateInterceptor(configuration.credentials)?.let { builder.addInterceptor(it) }
+            createAuthenticateInterceptor(configuration)?.let { builder.addInterceptor(it) }
         }
         builder.retryOnConnectionFailure(true)
         return builder.build()
@@ -263,9 +262,9 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
     /**
      * 创建身份认证拦截器
      */
-    private fun createAuthenticateInterceptor(configuration: RemoteCredentialsConfiguration): Interceptor? {
-        val username = configuration.username
-        val password = configuration.password
+    protected open fun createAuthenticateInterceptor(configuration: RemoteConfiguration): Interceptor? {
+        val username = configuration.credentials.username
+        val password = configuration.credentials.password
         return if (username != null && password != null) {
             BasicAuthInterceptor(username, password)
         } else null

src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/util/okhttp/PlatformAuthInterceptor.kt

@@ -0,0 +1,57 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.service.util.okhttp
+
+import com.tencent.bkrepo.common.api.constant.AUTH_HEADER_UID
+import com.tencent.bkrepo.common.api.constant.HttpHeaders
+import com.tencent.bkrepo.common.api.constant.PLATFORM_AUTH_PREFIX
+import okhttp3.Interceptor
+import okhttp3.Response
+import java.util.Base64
+
+/**
+ * 平台账号认证拦截器，用于添加平台认证凭据与用户ID到请求中
+ */
+class PlatformAuthInterceptor(accessKey: String, secretKey: String, private val userId: String) : Interceptor {
+
+    private val credentials = platform(accessKey, secretKey)
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        val authenticatedRequest = request.newBuilder()
+            .header(HttpHeaders.AUTHORIZATION, credentials)
+            .header(AUTH_HEADER_UID, userId)
+            .build()
+        return chain.proceed(authenticatedRequest)
+    }
+
+    fun platform(ak: String, sk: String): String {
+        val encoded = Base64.getEncoder().encodeToString("$ak:$sk".toByteArray())
+        return "$PLATFORM_AUTH_PREFIX$encoded"
+    }
+}

src/backend/generic/api-generic/src/main/kotlin/com/tencent/bkrepo/generic/constant/GenericMessageCode.kt

@@ -41,7 +41,8 @@ enum class GenericMessageCode(private val businessCode: Int, private val key: St
     LIST_DIR_NOT_ALLOWED(2, "generic.dir.not-allowed"),
     SIGN_FILE_NOT_FOUND(3, "generic.delta.sign-file.notfound"),
     NODE_DATA_HAS_CHANGED(4, "generic.node.data.has.changed"),
-    DOWNLOAD_DIR_NOT_ALLOWED(5, "generic.download.dir.not-allowed");
+    DOWNLOAD_DIR_NOT_ALLOWED(5, "generic.download.dir.not-allowed"),
+    ARTIFACT_SEARCH_FAILED(6, "generic.artifact.query.failed");
 
     override fun getBusinessCode() = businessCode
     override fun getKey() = key

src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericLocalRepository.kt

@@ -42,10 +42,13 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException
 import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException
 import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode
 import com.tencent.bkrepo.common.artifact.path.PathUtils
+import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory
 import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext
 import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository
 import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel
@@ -56,6 +59,7 @@ import com.tencent.bkrepo.common.artifact.stream.Range
 import com.tencent.bkrepo.common.service.util.HeaderUtils
 import com.tencent.bkrepo.common.service.util.HttpContextHolder
 import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext
 import com.tencent.bkrepo.generic.constant.BKREPO_META
 import com.tencent.bkrepo.generic.constant.BKREPO_META_PREFIX
 import com.tencent.bkrepo.generic.constant.GenericMessageCode
@@ -81,6 +85,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.NodeListOption
 import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
 import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest
+import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Component
 import org.springframework.util.unit.DataSize
@@ -414,6 +419,26 @@ class GenericLocalRepository(
         return redirectManager.redirect(context)
     }
 
+    override fun query(context: ArtifactQueryContext): Any? {
+        val artifactInfo = context.artifactInfo
+        return nodeClient.getNodeDetail(
+            artifactInfo.projectId,
+            artifactInfo.repoName,
+            artifactInfo.getArtifactFullPath()
+        ).data
+    }
+
+    override fun search(context: ArtifactSearchContext): List<Any> {
+        require(context is GenericArtifactSearchContext)
+        return context.queryModel?.let {
+            // 强制替换为请求的projectId与repoName避免越权
+            val newRule = replaceProjectIdAndRepo(it.rule, context.projectId, context.repoName)
+            nodeClient.queryWithoutCount(it.copy(rule = newRule)).data!!.records.onEach { node ->
+                (node as MutableMap<String, Any?>)[RepositoryInfo::category.name] = RepositoryCategory.LOCAL.name
+            }
+        } ?: emptyList()
+    }
+
     /**
      * 判断是否为分块上传
      */

src/backend/generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/artifact/GenericRemoteRepository.kt

@@ -31,13 +31,223 @@
 
 package com.tencent.bkrepo.generic.artifact
 
+import com.tencent.bkrepo.auth.constant.PIPELINE
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.api.constant.MediaTypes
+import com.tencent.bkrepo.common.api.exception.ErrorCodeException
+import com.tencent.bkrepo.common.api.pojo.Page
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.api.util.readJsonString
+import com.tencent.bkrepo.common.api.util.toJsonString
+import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory
+import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactSearchContext
 import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository
+import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter
+import com.tencent.bkrepo.common.query.enums.OperationType
+import com.tencent.bkrepo.common.query.model.Rule
+import com.tencent.bkrepo.common.security.util.SecurityUtils
+import com.tencent.bkrepo.common.service.util.okhttp.BasicAuthInterceptor
+import com.tencent.bkrepo.common.service.util.okhttp.PlatformAuthInterceptor
+import com.tencent.bkrepo.common.storage.innercos.http.toRequestBody
+import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext
+import com.tencent.bkrepo.generic.config.GenericProperties
+import com.tencent.bkrepo.generic.constant.GenericMessageCode
+import com.tencent.bkrepo.repository.pojo.node.NodeDetail
+import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Interceptor
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Request
+import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Component
 
 @Component
-class GenericRemoteRepository : RemoteRepository() {
+class GenericRemoteRepository(
+    private val genericProperties: GenericProperties
+) : RemoteRepository() {
     override fun onDownloadRedirect(context: ArtifactDownloadContext): Boolean {
         return redirectManager.redirect(context)
     }
+
+    override fun createAuthenticateInterceptor(configuration: RemoteConfiguration): Interceptor? {
+        val username = configuration.credentials.username
+        val password = configuration.credentials.password
+
+        // basic认证
+        if (!username.isNullOrEmpty() && !password.isNullOrEmpty()) {
+            return BasicAuthInterceptor(username, password)
+        }
+
+        // platform认证
+        val url = configuration.url.toHttpUrl()
+        genericProperties.platformAccounts.firstOrNull { it.host == url.host }?.let {
+            return PlatformAuthInterceptor(it.accessKey, it.secretKey, SecurityUtils.getUserId())
+        }
+
+        return null
+    }
+
+    override fun query(context: ArtifactQueryContext): Any? {
+        val remoteConfiguration = context.getRemoteConfiguration()
+        // 构造url
+        val (baseUrl, remoteProjectId, remoteRepoName) = splitBkRepoRemoteUrl(remoteConfiguration.url)
+        logger.info("query remoteProject[$remoteProjectId], remoteRepo[$remoteRepoName], user[${context.userId}]")
+
+        val artifactInfo = context.artifactInfo
+        val url = UrlFormatter.format(
+            baseUrl,
+            "repository/api/node/detail/$remoteProjectId/$remoteRepoName/${artifactInfo.getArtifactFullPath()}",
+            context.request.queryString
+        )
+
+        // 执行请求
+        val request = Request.Builder().url(url).get().build()
+        return request<Response<NodeDetail>>(context.getRemoteConfiguration(), request).data
+    }
+
+    override fun search(context: ArtifactSearchContext): List<Any> {
+        require(context is GenericArtifactSearchContext)
+        return context.queryModel?.let {
+            val remoteConfiguration = context.getRemoteConfiguration()
+
+            // 构造url
+            val (baseUrl, remoteProjectId, remoteRepoName) = splitBkRepoRemoteUrl(remoteConfiguration.url)
+            logger.info("search remoteProject[$remoteProjectId], remoteRepo[$remoteRepoName], user[${context.userId}]")
+
+            val result = if (firstSearch(context) && remoteRepoName == PIPELINE) {
+                searchPipelineNodes(context, baseUrl, remoteProjectId)
+            } else {
+                searchNodes(context, baseUrl, remoteProjectId, remoteRepoName)
+            }
+            result.onEach { node ->
+                (node as MutableMap<String, Any?>)[RepositoryInfo::category.name] = RepositoryCategory.REMOTE.name
+            }
+        } ?: emptyList()
+    }
+
+    /**
+     * 请求类似下方例子时，将作为前端首次进入仓库的请求
+     *
+     * {
+     *   ”projectId“: "xxx",
+     *   "repoName": "xxx",
+     *   "path": "/",
+     *   "folder": true // 可选
+     * }
+     */
+    private fun firstSearch(context: GenericArtifactSearchContext): Boolean {
+        var result = false
+        val rule = context.queryModel?.rule
+        if (rule is Rule.NestedRule) {
+            for (queryRule in rule.rules) {
+                result = false
+                if (queryRule !is Rule.QueryRule) {
+                    break
+                }
+
+                if (queryRule.field !in FIRST_SEARCH_FIELDS || queryRule.operation != OperationType.EQ) {
+                    break
+                }
+
+                if (queryRule.field == NodeDetail::path.name && queryRule.value != "/") {
+                    break
+                }
+                result = true
+            }
+
+        }
+        return result
+    }
+
+    private fun searchPipelineNodes(
+        context: GenericArtifactSearchContext,
+        baseUrl: String,
+        remoteProjectId: String
+    ): List<Any> {
+        // 构造url
+        val url = UrlFormatter.format(
+            baseUrl,
+            "repository/api/pipeline/list/$remoteProjectId",
+            context.request.queryString
+        )
+        val request = Request.Builder().get().url(url).build()
+        return request<Response<List<Map<String, Any?>>>>(context.getRemoteConfiguration(), request).data!!
+    }
+
+    private fun searchNodes(
+        context: GenericArtifactSearchContext,
+        baseUrl: String,
+        remoteProjectId: String,
+        remoteRepoName: String
+    ): List<Any> {
+        // 构造url
+        val url = UrlFormatter.format(
+            baseUrl,
+            "repository/api/node/queryWithoutCount",
+            context.request.queryString
+        )
+
+        // 构造body
+        val newRule = replaceProjectIdAndRepo(context.queryModel!!.rule, remoteProjectId, remoteRepoName)
+        val newQueryModel = context.queryModel!!.copy(rule = newRule)
+        val body = newQueryModel.toJsonString().toRequestBody(MediaTypes.APPLICATION_JSON.toMediaType())
+
+        // 执行请求
+        val request = Request.Builder().url(url).post(body).build()
+        return request<Response<Page<Map<String, Any?>>>>(context.getRemoteConfiguration(), request).data!!.records
+    }
+
+    private inline fun <reified T> request(remoteConfiguration: RemoteConfiguration, request: Request): T {
+        val httpClient = createHttpClient(remoteConfiguration)
+        val response = httpClient.newCall(request).execute()
+        // 解析结果
+        return if (response.isSuccessful) {
+            response.body!!.byteStream().use { it.readJsonString<T>() }
+        } else {
+            val msg = response.body?.string()
+            logger.warn("request failed, url[${request.url}], code[${response.code}], msg[$msg]")
+            throw ErrorCodeException(
+                status = HttpStatus.BAD_REQUEST,
+                messageCode = GenericMessageCode.ARTIFACT_SEARCH_FAILED,
+                params = arrayOf("remote response code[${response.code}]")
+            )
+        }
+    }
+
+    /**
+     * 解析出BkRepo Url 的projectId和repoName
+     * 比如http://bkrepo.example.com/projectId/repoName将会返回(http://bkrepo.example.com, projectId, repoName)
+     */
+    private fun splitBkRepoRemoteUrl(url: String): Triple<String, String, String> {
+        val httpUrl = url.trimEnd('/').toHttpUrl()
+        val builder = httpUrl.newBuilder()
+        if (httpUrl.pathSize < 2 || httpUrl.pathSegments[0].isEmpty() || httpUrl.pathSegments[1].isEmpty()) {
+            throw ErrorCodeException(
+                messageCode = GenericMessageCode.ARTIFACT_SEARCH_FAILED,
+                status = HttpStatus.BAD_REQUEST,
+                params = arrayOf("failed to split remote url[${url}]")
+            )
+        }
+        for (i in 0 until httpUrl.pathSize) {
+            builder.removePathSegment(0)
+        }
+        return Triple(
+            builder.build().toString(),
+            httpUrl.pathSegments[httpUrl.pathSize - 2],
+            httpUrl.pathSegments[httpUrl.pathSize - 1]
+        )
+    }
+
+    companion object {
+        private val FIRST_SEARCH_FIELDS = listOf(
+            NodeDetail::projectId.name,
+            NodeDetail::repoName.name,
+            NodeDetail::path.name,
+            NodeDetail::folder.name
+        )
+        private val logger = LoggerFactory.getLogger(GenericRemoteRepository::class.java)
+    }
 }