Skip to content

Commit

Permalink
refactor: npm远程仓库优化 #1168
Browse files Browse the repository at this point in the history
  • Loading branch information
@scplsy
scplsy committed Sep 15, 2023
1 parent 33f46e0 commit 715f984
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 65 deletions.
14 changes: 6 additions & 8 deletions .../src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,21 +115,19 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
/**
* 尝试读取缓存的远程构件
*/
fun getCacheArtifactResource(context: ArtifactDownloadContext): ArtifactResource? {
open fun getCacheArtifactResource(context: ArtifactContext): ArtifactResource? {
val configuration = context.getRemoteConfiguration()
if (!configuration.cache.enabled) return null

val cacheNode = findCacheNodeDetail(context)
if (cacheNode == null || cacheNode.folder) return null
return if (!isExpired(cacheNode, configuration.cache.expiration)) {
loadArtifactResource(cacheNode, context)
} else null
return if (cacheNode == null || cacheNode.folder || isExpired(cacheNode, configuration.cache.expiration)) null
else loadArtifactResource(cacheNode, context)
}

/**
* 加载要返回的资源
*/
open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactContext): ArtifactResource? {
return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run {
if (logger.isDebugEnabled) {
logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
Expand All @@ -152,7 +150,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
/**
* 尝试获取缓存的远程构件节点
*/
open fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? {
open fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? {
with(context) {
return nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data
}
Expand All @@ -177,7 +175,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
val size = artifactFile.getSize()
val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size))
val node = cacheArtifactFile(context, artifactFile)
return ArtifactResource(artifactStream, context.artifactInfo.getResponseName(), node, ArtifactChannel.LOCAL)
return ArtifactResource(artifactStream, context.artifactInfo.getResponseName(), node, ArtifactChannel.PROXY)
}

/**
Expand Down
4 changes: 2 additions & 2 deletions src/backend/npm/api-npm/src/main/kotlin/com/tencent/bkrepo/npm/constants/Constants.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ const val NPM_PKG_VERSION_METADATA_FULL_PATH = "/.npm/%s/%s-%s.json"
const val NPM_PKG_METADATA_FULL_PATH = "/.npm/%s/package.json"

const val NPM_FILE_FULL_PATH = "npm_file_full_path"

const val SEARCH_REQUEST = "search_request"

const val PKG_NAME = "pkg_name"
const val REQUEST_URI = "requestURI"
const val PACKAGE_JSON = "package.json"

// constants map
val ERROR_MAP = mapOf("error" to "not_found", "reason" to "document not found")
Expand Down
98 changes: 46 additions & 52 deletions ...biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

package com.tencent.bkrepo.npm.artifact.repository

import com.tencent.bkrepo.common.api.constant.MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET
import com.tencent.bkrepo.common.api.util.JsonUtils
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext
Expand All @@ -45,12 +46,14 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter
import com.tencent.bkrepo.common.storage.monitor.Throughput
import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH
import com.tencent.bkrepo.npm.constants.PACKAGE_JSON
import com.tencent.bkrepo.npm.constants.REQUEST_URI
import com.tencent.bkrepo.npm.exception.NpmBadRequestException
import com.tencent.bkrepo.npm.pojo.NpmSearchInfoMap
import com.tencent.bkrepo.npm.pojo.NpmSearchResponse
import com.tencent.bkrepo.npm.utils.NpmUtils
import com.tencent.bkrepo.repository.pojo.node.NodeDetail
import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
import okhttp3.Request
import okhttp3.Response
import org.slf4j.Logger
import org.slf4j.LoggerFactory
Expand All @@ -69,34 +72,13 @@ class NpmRemoteRepository(
throughput: Throughput
) {
super.onDownloadSuccess(context, artifactResource, throughput)
val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(context.artifactInfo.getArtifactFullPath())
val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second)
val queryContext = ArtifactQueryContext(context.repo, context.artifactInfo)
queryContext.putAttribute(NPM_FILE_FULL_PATH, versionMetadataFullPath)
queryContext.putAttribute(REQUEST_URI, "/${packageInfo.first}/${packageInfo.second}")
// 存储package-version.json文件
executor.execute { cachePackageVersionMetadata(context) }
}

private fun cachePackageVersionMetadata(context: ArtifactDownloadContext) {
with(context) {
val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(artifactInfo.getArtifactFullPath())
val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second)
if (nodeClient.checkExist(projectId, repoName, versionMetadataFullPath).data!!) {
logger.info(
"version metadata [$versionMetadataFullPath] is already exits " +
"in repo [$projectId/$repoName]"
)
return
}
val remoteConfiguration = context.getRemoteConfiguration()
val httpClient = createHttpClient(remoteConfiguration)
context.putAttribute("requestURI", "/${packageInfo.first}/${packageInfo.second}")
val downloadUri = createRemoteSearchUrl(context)
val request = Request.Builder().url(downloadUri).build()
val response = httpClient.newCall(request).execute()
if (checkResponse(response)) {
val artifactFile = createTempFile(response.body!!)
context.putAttribute(NPM_FILE_FULL_PATH, versionMetadataFullPath)
cacheArtifactFile(context, artifactFile)
logger.info("cache version metadata [$versionMetadataFullPath] success.")
}
}
executor.execute { findCacheNodeDetail(queryContext) ?: (super.query(queryContext) as InputStream?)?.close() }
}

override fun upload(context: ArtifactUploadContext) {
Expand All @@ -108,42 +90,54 @@ class NpmRemoteRepository(
}

override fun query(context: ArtifactQueryContext): InputStream? {
val remoteConfiguration = context.getRemoteConfiguration()
val httpClient = createHttpClient(remoteConfiguration)
val downloadUri = createRemoteSearchUrl(context)
val request = Request.Builder().url(downloadUri).build()
val response = httpClient.newCall(request).execute()
return if (checkResponse(response)) {
onQueryResponse(context, response)
} else null
return getCacheArtifactResource(context)?.getSingleStream() ?: super.query(context) as InputStream?
}

private fun createRemoteSearchUrl(context: ArtifactContext): String {
override fun checkQueryResponse(response: Response): Boolean {
return super.checkQueryResponse(response) && run {
val contentType = response.body!!.contentType()
contentType.toString().contains(APPLICATION_JSON_WITHOUT_CHARSET) || run {
logger.warn("Content-Type($contentType) of response from [${response.request.url}] is unsupported")
false
}
}
}

// 仅package.json文件有必要在缓存过期后更新
override fun getCacheArtifactResource(context: ArtifactContext): ArtifactResource? {
return when (context) {
is ArtifactDownloadContext -> findCacheNodeDetail(context)?.let { loadArtifactResource(it, context) }
is ArtifactQueryContext -> {
if (context.getStringAttribute(NPM_FILE_FULL_PATH)?.endsWith("/$PACKAGE_JSON") == false) {
findCacheNodeDetail(context)?.let { loadArtifactResource(it, context) }
} else if (context.getRemoteConfiguration().cache.expiration > 0) {
super.getCacheArtifactResource(context)
} else null
}
else -> null
}
}

override fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? {
val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!!
with(context) {
return nodeClient.getNodeDetail(projectId, repoName, fullPath).data
}
}

override fun createRemoteDownloadUrl(context: ArtifactContext): String {
val configuration = context.getRemoteConfiguration()
val requestURI = context.getStringAttribute("requestURI")
val requestURI = context.getStringAttribute(REQUEST_URI)
val artifactUri =
requestURI ?: context.request.requestURI.substringAfterLast(context.artifactInfo.getRepoIdentify())
val queryString = context.request.queryString
return UrlFormatter.format(configuration.url, artifactUri, queryString)
}

override fun onQueryResponse(context: ArtifactQueryContext, response: Response): InputStream? {
val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!!
val body = response.body!!
val artifactFile = createTempFile(body)
val sha256 = artifactFile.getFileSha256()
with(context) {
nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let {
if (it.sha256.equals(sha256)) {
logger.info("artifact [$fullPath] is hit the cache.")
return artifactFile.getInputStream()
}
cacheArtifactFile(context, artifactFile)
} ?: run {
// 存储构件
cacheArtifactFile(context, artifactFile)
}
}
cacheArtifactFile(context, artifactFile)
return artifactFile.getInputStream()
}

Expand Down
3 changes: 3 additions & 0 deletions ...d/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import com.tencent.bkrepo.npm.constants.LATEST
import com.tencent.bkrepo.npm.constants.MODIFIED
import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH
import com.tencent.bkrepo.npm.constants.NPM_PACKAGE_TGZ_FILE
import com.tencent.bkrepo.npm.constants.REQUEST_URI
import com.tencent.bkrepo.npm.constants.SEARCH_REQUEST
import com.tencent.bkrepo.npm.constants.SIZE
import com.tencent.bkrepo.npm.exception.NpmArtifactExistException
Expand Down Expand Up @@ -275,6 +276,7 @@ class NpmClientServiceImpl(
val context = ArtifactQueryContext()
val packageFullPath = NpmUtils.getPackageMetadataPath(name)
context.putAttribute(NPM_FILE_FULL_PATH, packageFullPath)
context.putAttribute(REQUEST_URI, name)
val inputStream =
ArtifactContextHolder.getRepository().query(context) as? InputStream
?: throw NpmArtifactNotFoundException("document not found")
Expand Down Expand Up @@ -304,6 +306,7 @@ class NpmClientServiceImpl(
val context = ArtifactQueryContext()
val packageFullPath = NpmUtils.getVersionPackageMetadataPath(name, version)
context.putAttribute(NPM_FILE_FULL_PATH, packageFullPath)
context.putAttribute(REQUEST_URI, "$name/$version")
val inputStream =
ArtifactContextHolder.getRepository().query(context) as? InputStream
?: throw NpmArtifactNotFoundException("document not found")
Expand Down
6 changes: 3 additions & 3 deletions ...src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ class OciRegistryRemoteRepository(
/**
* 尝试获取缓存的远程构件节点
*/
override fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? {
override fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? {
with(context) {
val fullPath = ociOperationService.getNodeFullPath(context.artifactInfo as OciArtifactInfo) ?: return null
return nodeClient.getNodeDetail(projectId, repoName, fullPath).data
Expand All @@ -463,7 +463,7 @@ class OciRegistryRemoteRepository(
/**
* 加载要返回的资源: oci协议需要返回特定的请求头和资源类型
*/
override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactContext): ArtifactResource? {
return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run {
if (logger.isDebugEnabled) {
logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
Expand All @@ -484,7 +484,7 @@ class OciRegistryRemoteRepository(

private fun buildResponse(
cacheNode: NodeDetail?,
context: ArtifactDownloadContext,
context: ArtifactContext,
artifactResource: ArtifactResource,
sha256: String? = null,
size: Long? = null
Expand Down

0 comments on commit 715f984

Please sign in to comment.