diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index c4e56a21fe..f9cd952b0f 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -37,6 +37,9 @@ const val DOCKER_API_VERSION = "registry/2.0" const val DOCKER_CONTENT_DIGEST = "Docker-Content-Digest" const val DOCKER_UPLOAD_UUID = "Docker-Upload-Uuid" const val BLOB_UPLOAD_SESSION_ID = "Blob-Upload-Session-ID" +const val ARCHITECTURE = "architecture" +const val OS = "os" +const val VARIANT = "variant" const val HTTP_FORWARDED_PROTO = "X-Forwarded-Proto" const val HTTP_PROTOCOL_HTTP = "http" @@ -62,6 +65,7 @@ const val DIGEST = "oci_digest" const val SIZE = "size" const val SCHEMA_VERSION = "schemaVersion" const val IMAGE_VERSION = "blob_version" +const val OLD_DOCKER_VERSION = "docker.manifest" const val MANIFEST_DIGEST = "manifest_digest" const val DIGEST_LIST = "digest_list" const val FORCE = "force" @@ -101,12 +105,16 @@ const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca const val OCI_MANIFEST = "manifest.json" +const val OCI_MANIFEST_LIST = "list.manifest.json" const val STAGE_TAG = "stageTag" const val TAG_LIST_REQUEST = "tagList" const val CATALOG_REQUEST = "catalog" const val REQUEST_IMAGE = "image" +const val DOCKER_REPO_NAME = "docker.repoName" +const val DOCKER_MANIFEST_DIGEST = "docker.manfiest.digest" + // OCIScheme is the URL scheme for OCI-based requests const val OCI_SCHEME = "oci" @@ -129,6 +137,8 @@ const val OCI_IMAGE_MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1 const val DOCKER_DISTRIBUTION_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json" +const val DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 = "application/vnd.docker.distribution.manifest.list.v2+json" + // Content Descriptor const val CONTENT_DESCRIPTOR_MEDIA_TYPE = "application/vnd.oci.descriptor.v1+json" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt new file mode 100644 index 0000000000..538198e472 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt @@ -0,0 +1,16 @@ +package com.tencent.bkrepo.oci.model + +import com.fasterxml.jackson.annotation.JsonProperty + +data class ConfigSchema2( + var architecture: String, + var history: List, + var os: String, + val variant: String? = null +) + +data class History( + var created: String, + @JsonProperty("created_by") + var createdBy: String? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt new file mode 100644 index 0000000000..8013ff995e --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.oci.model + +data class ManifestDescriptor( + override var mediaType: String, + override var size: Long, + override var digest: String, + var platform: Map = emptyMap() +) : Descriptor(mediaType, size, digest) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt new file mode 100644 index 0000000000..6d4a46044c --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt @@ -0,0 +1,7 @@ +package com.tencent.bkrepo.oci.model + +class ManifestList( + override var schemaVersion: Int, + var mediaType: String, + var manifests: List, +) : SchemaVersion(schemaVersion) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt index 0314c8a54e..7e787713df 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt @@ -27,6 +27,8 @@ package com.tencent.bkrepo.oci.pojo.artifact +import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 +import com.tencent.bkrepo.oci.constant.IMAGE_INDEX_MEDIA_TYPE import com.tencent.bkrepo.oci.util.OciLocationUtils class OciManifestArtifactInfo( @@ -35,17 +37,28 @@ class OciManifestArtifactInfo( packageName: String, version: String, val reference: String, - val isValidDigest: Boolean + val isValidDigest: Boolean, + var mediaType: String? = null ) : OciArtifactInfo(projectId, repoName, packageName, version) { override fun getArtifactFullPath(): String { return if(getArtifactMappingUri().isNullOrEmpty()) { if (isValidDigest) { - OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference) + OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference, isFatManifest()) + } else if (isFatManifest()) { + OciLocationUtils.buildManifestListPath(packageName, reference) } else { OciLocationUtils.buildManifestPath(packageName, reference) } } else getArtifactMappingUri()!! + } + + fun isFatManifest() = FAT_MANIFEST_MEDIA_TYPES.contains(mediaType) + companion object { + private val FAT_MANIFEST_MEDIA_TYPES = listOf( + DOCKER_DISTRIBUTION_MANIFEST_LIST_V2, + IMAGE_INDEX_MEDIA_TYPE + ) } } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt index cc4849fca2..19715bf5e1 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt @@ -58,5 +58,7 @@ data class BasicInfo( @ApiModelProperty("修改者") val lastModifiedBy: String, @ApiModelProperty("修改时间") - val lastModifiedDate: String + val lastModifiedDate: String, + @ApiModelProperty("操作系统和架构信息") + val platform: List ) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt index 698937409f..f07319fc59 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt @@ -40,6 +40,8 @@ data class PackageVersionInfo( @ApiModelProperty("基础信息") val basic: BasicInfo, @ApiModelProperty("元数据信息") - val metadata: List + val metadata: List, + @ApiModelProperty("history") + val history: List ) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index c7b1dbb7c2..92d84a3d54 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.oci.util import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.oci.constant.OCI_MANIFEST_LIST import com.tencent.bkrepo.oci.constant.OCI_MANIFEST import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest @@ -38,6 +39,10 @@ object OciLocationUtils { return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST } + fun buildManifestListPath(packageName: String, tag: String): String { + return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST_LIST + } + fun buildManifestVersionFolderPath(packageName: String, tag: String): String { return buildManifestFolderPath(packageName) +"$tag/" } @@ -46,8 +51,9 @@ object OciLocationUtils { return "/$packageName/manifest/" } - fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { - return buildDigestManifestPath(packageName, OciDigest(reference)) + fun buildDigestManifestPathWithReference(packageName: String, reference: String, isFatManifest: Boolean): String { + return buildManifestVersionFolderPath(packageName, reference) + + if (isFatManifest) OCI_MANIFEST_LIST else OCI_MANIFEST } private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt index 29004f80ff..39f7315312 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.model.Descriptor +import com.tencent.bkrepo.oci.model.ManifestList import com.tencent.bkrepo.oci.model.ManifestSchema1 import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.SchemaVersion @@ -88,6 +89,14 @@ object OciUtils { } } + fun stringToManifestList(content: String): ManifestList { + try { + return content.readJsonString() + } catch (e: Exception) { + throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_INVALID, Strings.EMPTY) + } + } + fun manifestIterator(manifest: ManifestSchema2): List { val list = mutableListOf() list.add(manifest.config) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index cb8212f64c..0c0be78e65 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -57,6 +57,7 @@ import com.tencent.bkrepo.oci.constant.MEDIA_TYPE import com.tencent.bkrepo.oci.constant.N import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.OLD_DOCKER_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.OLD_DOCKER_VERSION import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PATCH import com.tencent.bkrepo.oci.constant.POST @@ -335,7 +336,9 @@ class OciRegistryLocalRepository( if (context.request.method == HttpMethod.HEAD.name) { return null } - val version = artifactResource.node?.metadata?.get(IMAGE_VERSION)?.toString() ?: return null + val version = artifactResource.node?.metadata?.get(IMAGE_VERSION)?.toString() ?: run { + artifactResource.node?.metadata?.get(OLD_DOCKER_VERSION)?.toString() ?: return null + } return PackageDownloadRecord( projectId = context.projectId, repoName = context.repoName, @@ -374,7 +377,7 @@ class OciRegistryLocalRepository( } logger.info( - "The mediaType of Artifact $fullPath is $mediaType and it's contentType is $contentType" + + "The mediaType of Artifact ${node.fullPath} is $mediaType and it's contentType is $contentType" + "in repo: ${context.artifactInfo.getRepoIdentify()}" ) OciResponseUtils.buildDownloadResponse( @@ -400,7 +403,15 @@ class OciRegistryLocalRepository( val oldDockerPath = ociOperationService.getDockerNode(artifactInfo) ?: return null artifactInfo.setArtifactMappingUri(oldDockerPath) - ArtifactContextHolder.getNodeDetail(artifactInfo) + ArtifactContextHolder.getNodeDetail(artifactInfo) ?: run { + if (artifactInfo !is OciManifestArtifactInfo) return null + // 兼容 list.manifest.json + val manifestListPath = OciLocationUtils.buildManifestListPath( + artifactInfo.packageName, artifactInfo.reference + ) + artifactInfo.setArtifactMappingUri(manifestListPath) + ArtifactContextHolder.getNodeDetail(artifactInfo) + } } } @@ -446,7 +457,9 @@ class OciRegistryLocalRepository( with(context) { val artifactInfo = context.artifactInfo as OciArtifactInfo val packageKey = PackageKeys.ofName(repo.type, artifactInfo.packageName) - val version = node.metadata[IMAGE_VERSION]?.toString() ?: return null + val version = node.metadata[IMAGE_VERSION]?.toString() ?: run { + node.metadata[OLD_DOCKER_VERSION]?.toString() ?: return null + } return packageClient.findVersionByName(projectId, repoName, packageKey, version).data } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 0991892b88..0e227cf814 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -430,13 +430,17 @@ class OciRegistryRemoteRepository( */ override fun onDownloadResponse(context: ArtifactDownloadContext, response: Response): ArtifactResource { logger.info("Remote download response will be processed") + val ociArtifactInfo = context.artifactInfo + if (ociArtifactInfo is OciManifestArtifactInfo) { + ociArtifactInfo.mediaType = response.header(HttpHeaders.CONTENT_TYPE) + } val artifactFile = createTempFile(response.body!!) val size = artifactFile.getSize() val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size)) val node = cacheArtifact(context, artifactFile) val artifactResource = ArtifactResource( inputStream = artifactStream, - artifactName = context.artifactInfo.getResponseName(), + artifactName = ociArtifactInfo.getResponseName(), node = node, channel = ArtifactChannel.PROXY ) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt index 68dfb1894a..3bc4edfdc5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.oci.artifact.resolver +import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder @@ -69,7 +70,10 @@ class OciManifestArtifactInfoResolver : ArtifactInfoResolver { val reference = attributes["reference"].toString().trim() validate(packageName) val isValidDigest = OciDigest.isValid(reference) - OciManifestArtifactInfo(projectId, repoName, packageName, "", reference, isValidDigest) + val contentType = request.getHeader(HttpHeaders.CONTENT_TYPE) + OciManifestArtifactInfo( + projectId, repoName, packageName, "", reference, isValidDigest, contentType + ) } } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt index 88aac6b016..a73602edc6 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt @@ -44,6 +44,8 @@ import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.LocaleMessageUtils import com.tencent.bkrepo.oci.artifact.auth.OciLoginAuthHandler import com.tencent.bkrepo.oci.config.OciProperties +import com.tencent.bkrepo.oci.constant.DOCKER_API_VERSION +import com.tencent.bkrepo.oci.constant.DOCKER_HEADER_API_VERSION import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_CODE import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_DESCRIPTION import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_MESSAGE @@ -72,6 +74,7 @@ class OciExceptionHandler( fun handleException(exception: AuthenticationException) { val response = HttpContextHolder.getResponse() response.contentType = MediaTypes.APPLICATION_JSON + response.setHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) response.addHeader( HttpHeaders.WWW_AUTHENTICATE, OciLoginAuthHandler.AUTH_CHALLENGE_SERVICE_SCOPE.format( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 53646481db..9d8f19182e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -73,9 +73,13 @@ class OciBlobServiceImpl( override fun startUploadBlob(artifactInfo: OciBlobArtifactInfo, artifactFile: ArtifactFile) { with(artifactInfo) { - logger.info("Handling bolb upload request $artifactInfo in ${getRepoIdentify()} .") + logger.info( + "Handling bolb upload request ${artifactInfo.digest}|${artifactInfo.uuid}|" + + "${artifactInfo.mount}|${artifactInfo.from} in ${getRepoIdentify()}." + ) if (digest.isNullOrBlank()) { logger.info("Will use post then put to upload blob...") + // docker manifest mount upload blob obtainSessionIdForUpload(artifactInfo) } else { logger.info("Will use single post to upload blob...") diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index a2379d8bdd..db0c7f106a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -28,8 +28,9 @@ package com.tencent.bkrepo.oci.service.impl import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER -import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.constant.ensurePrefix +import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE @@ -48,15 +49,17 @@ import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties +import com.tencent.bkrepo.oci.constant.ARCHITECTURE import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.DESCRIPTION +import com.tencent.bkrepo.oci.constant.DOCKER_MANIFEST_DIGEST +import com.tencent.bkrepo.oci.constant.DOCKER_REPO_NAME import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_DATE @@ -64,19 +67,27 @@ import com.tencent.bkrepo.oci.constant.MANIFEST_DIGEST import com.tencent.bkrepo.oci.constant.MD5 import com.tencent.bkrepo.oci.constant.NODE_FULL_PATH import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.OCI_MANIFEST_LIST import com.tencent.bkrepo.oci.constant.OCI_NODE_FULL_PATH import com.tencent.bkrepo.oci.constant.OCI_NODE_SIZE import com.tencent.bkrepo.oci.constant.OCI_PACKAGE_NAME +import com.tencent.bkrepo.oci.constant.OLD_DOCKER_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.OLD_DOCKER_VERSION +import com.tencent.bkrepo.oci.constant.OS import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL import com.tencent.bkrepo.oci.constant.REPO_TYPE +import com.tencent.bkrepo.oci.constant.VARIANT import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension import com.tencent.bkrepo.oci.extension.ImagePackagePullContext +import com.tencent.bkrepo.oci.model.ConfigSchema2 import com.tencent.bkrepo.oci.model.Descriptor +import com.tencent.bkrepo.oci.model.History +import com.tencent.bkrepo.oci.model.ManifestList import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.TOciReplicationRecord import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo @@ -95,6 +106,7 @@ import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciLocationUtils.buildBlobsFolderPath import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.oci.util.OciUtils +import com.tencent.bkrepo.replication.constant.SHA256 import com.tencent.bkrepo.repository.api.MetadataClient import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient @@ -298,11 +310,52 @@ class OciOperationServiceImpl( OciMessageCode.OCI_VERSION_NOT_FOUND, "$packageKey/$version", "$projectId|$repoName" ) val packageVersion = packageClient.findVersionByName(projectId, repoName, packageKey, version).data!! - val basicInfo = ObjectBuildUtils.buildBasicInfo(nodeDetail, packageVersion) - return PackageVersionInfo(basicInfo, packageVersion.packageMetadata) + val pair = getManifestInfo(nodeDetail, repoDetail, name) + val basicInfo = ObjectBuildUtils.buildBasicInfo(nodeDetail, packageVersion, pair.first) + return PackageVersionInfo(basicInfo, packageVersion.packageMetadata, pair.second) } } + private fun getManifestInfo( + nodeDetail: NodeDetail, + repoDetail: RepositoryDetail, + packageName: String + ): Pair, List> { + val platform = mutableListOf() + var history = listOf() + if (nodeDetail.name == OCI_MANIFEST_LIST) { + // manifestList + loadManifestList( + nodeDetail.sha256!!, + nodeDetail.size, + repoDetail.storageCredentials + )?.let { manifestList -> + manifestList.manifests.forEach { manifest -> + val map = manifest.platform + platform.add( + "${map[OS]}/${map[ARCHITECTURE]}" + (map[VARIANT]?.toString()?.ensurePrefix("/") ?: "") + ) + } + } + } else { + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, repoDetail.storageCredentials) + // config + manifest?.config?.digest?.let { configDigest -> + val configNode = getImageNodeDetail( + nodeDetail.projectId, + nodeDetail.repoName, + packageName, + configDigest + ) + val inputStream = storageManager.loadArtifactInputStream(configNode, repoDetail.storageCredentials) + val config = JsonUtils.objectMapper.readValue(inputStream, ConfigSchema2::class.java) + platform.add("${config.os}/${config.architecture}" + (config.variant?.ensurePrefix("/") ?: "")) + history = config.history + } + } + return Pair(platform, history) + } + /** * 获取node节点 * 查不到抛出OciFileNotFoundException异常 @@ -338,10 +391,15 @@ class OciOperationServiceImpl( val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: run { val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return@run null nodeClient.getNodeDetail(projectId, repoName, oldDockerFullPath).data ?: run { - logger.warn("node [$fullPath] don't found.") - null + if (ociArtifactInfo !is OciManifestArtifactInfo) return null + // 兼容 list.manifest.json + val manifestListPath = OciLocationUtils.buildManifestListPath( + ociArtifactInfo.packageName, ociArtifactInfo.reference + ) + nodeClient.getNodeDetail(projectId, repoName, manifestListPath).data } } + logger.info("Get images node fullPath:$fullPath, nodeDetail fullPath:${nodeDetail?.fullPath}") return nodeDetail } @@ -429,36 +487,59 @@ class OciOperationServiceImpl( // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go // docker 客户端上传manifest时先按照schema2的格式上传， // 如失败则按照schema1格式上传，但是非docker客户端不兼容schema1版本manifest - val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) - ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) - // 更新manifest文件的metadata - val mediaType = if (manifest.mediaType.isNullOrEmpty()) { - HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + // 需要区分 manifest.json 和 list.manifest.json + if (ociArtifactInfo.isFatManifest()) { + val mediaType = loadManifestList(nodeDetail.sha256!!, nodeDetail.size, storageCredentials)!!.mediaType + // 更新manifest节点元数据 + updateNodeMetaData( + projectId = ociArtifactInfo.projectId, + repoName = ociArtifactInfo.repoName, + version = ociArtifactInfo.reference, + fullPath = nodeDetail.fullPath, + mediaType = mediaType, + sourceType = sourceType + ) + val metadata = mutableMapOf( + OLD_DOCKER_VERSION to ociArtifactInfo.reference, + SHA256 to nodeDetail.sha256!!, + DOCKER_REPO_NAME to ociArtifactInfo.packageName, + DOCKER_MANIFEST_DIGEST to OciDigest.fromSha256(nodeDetail.sha256!!).toString(), + OLD_DOCKER_MEDIA_TYPE to mediaType, + ) + doPackageOperations( + manifestPath = nodeDetail.fullPath, + ociArtifactInfo = ociArtifactInfo, + manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), + size = nodeDetail.size, + sourceType = sourceType, + metadata = metadata, + userId = SecurityUtils.getUserId() + ) } else { - manifest.mediaType + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) + ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) + val mediaType = + manifest.mediaType?.ifEmpty { null } ?: ociArtifactInfo.mediaType ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + val digestList = OciUtils.manifestIteratorDigest(manifest) + // 更新manifest节点元数据 + updateNodeMetaData( + projectId = ociArtifactInfo.projectId, + repoName = ociArtifactInfo.repoName, + version = ociArtifactInfo.reference, + fullPath = nodeDetail.fullPath, + mediaType = mediaType, + digestList = digestList, + sourceType = sourceType + ) + if (ociArtifactInfo.packageName.isEmpty()) return + // 处理manifest中的blob数据 + syncBlobInfo( + ociArtifactInfo = ociArtifactInfo, + manifest = manifest, + nodeDetail = nodeDetail, + sourceType = sourceType + ) } - val digestList = OciUtils.manifestIteratorDigest(manifest) - - // 更新manifest节点元数据 - updateNodeMetaData( - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - version = ociArtifactInfo.reference, - fullPath = nodeDetail.fullPath, - mediaType = mediaType!!, - digestList = digestList, - sourceType = sourceType - ) - - - if (ociArtifactInfo.packageName.isEmpty()) return - // 处理manifest中的blob数据 - syncBlobInfo( - ociArtifactInfo = ociArtifactInfo, - manifest = manifest, - nodeDetail = nodeDetail, - sourceType = sourceType - ) } @@ -476,6 +557,26 @@ class OciOperationServiceImpl( OciUtils.stringToManifestV2(manifestBytes) } catch (e: Exception) { + logger.error("load manifest error, sha256:$sha256") + null + } + } + + private fun loadManifestList( + sha256: String, + size: Long, + storageCredentials: StorageCredentials? + ): ManifestList? { + return try { + val manifestBytes = storageService.load( + sha256, + Range.full(size), + storageCredentials + )!!.readText() + + OciUtils.stringToManifestList(manifestBytes) + } catch (e: Exception) { + logger.error("load manifest list error, sha256:$sha256") null } } @@ -572,6 +673,15 @@ class OciOperationServiceImpl( ) // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 if (existFlag) { + val mediaType = + manifest.mediaType?.ifEmpty { null } ?: ociArtifactInfo.mediaType ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + val metadata = mutableMapOf( + OLD_DOCKER_VERSION to ociArtifactInfo.reference, + SHA256 to nodeDetail.sha256!!, + DOCKER_REPO_NAME to ociArtifactInfo.packageName, + DOCKER_MANIFEST_DIGEST to OciDigest.fromSha256(nodeDetail.sha256!!).toString(), + OLD_DOCKER_MEDIA_TYPE to mediaType + ) // 第三方同步的索引更新等所有文件全部上传完成后才去进行 // 根据flag生成package信息以及package version信息 doPackageOperations( @@ -580,6 +690,7 @@ class OciOperationServiceImpl( manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), size = size, sourceType = sourceType, + metadata = metadata, userId = userId ) return true @@ -708,6 +819,7 @@ class OciOperationServiceImpl( manifestDigest: OciDigest, size: Long, sourceType: ArtifactChannel? = null, + metadata: MutableMap, userId: String = SecurityUtils.getUserId() ) { with(ociArtifactInfo) { @@ -715,10 +827,8 @@ class OciOperationServiceImpl( // 针对支持多仓库类型，如docker和oci val repoType = repositoryClient.getRepoDetail(projectId, repoName).data!!.type.name val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) - val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) - .apply { - sourceType?.let { this[SOURCE_TYPE] = sourceType } - } + metadata[MANIFEST_DIGEST] = manifestDigest.toString() + sourceType?.let { metadata.put(SOURCE_TYPE, sourceType) } val request = ObjectBuildUtils.buildPackageVersionCreateRequest( ociArtifactInfo = this, packageName = packageName, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index c7a3c534d8..76618d3d3d 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -212,7 +212,7 @@ object ObjectBuildUtils { ) } - fun buildBasicInfo(nodeDetail: NodeDetail, packageVersion: PackageVersion): BasicInfo { + fun buildBasicInfo(nodeDetail: NodeDetail, packageVersion: PackageVersion, platform: List): BasicInfo { with(nodeDetail) { return BasicInfo( version = packageVersion.name, @@ -227,7 +227,8 @@ object ObjectBuildUtils { createdBy = createdBy, createdDate = createdDate, lastModifiedBy = lastModifiedBy, - lastModifiedDate = lastModifiedDate + lastModifiedDate = lastModifiedDate, + platform = platform ) } }