feat: 支持在缓存被保留时也触发预加载计划生成 TencentBlueKing#2607

* feat: 支持在缓存被保留时也触发预加载计划生成 TencentBlueKing#2607 * feat: 支持在缓存被保留时也触发预加载计划生成 TencentBlueKing#2607 * feat: 支持在缓存被保留时也触发预加载计划生成 TencentBlueKing#2607 * feat: 支持在缓存被保留时也触发预加载计划生成 TencentBlueKing#2607 * feat: 支持在缓存被保留时也触发预加载计划生成 TencentBlueKing#2607
scplsy · Sep 26, 2024 · 9a30d5a · 9a30d5a
1 parent b79129f
commit 9a30d5a
Show file tree

Hide file tree

Showing 6 changed files with 131 additions and 21 deletions.
diff --git a/.../main/kotlin/com/tencent/bkrepo/common/artifact/cache/config/ArtifactPreloadProperties.kt b/.../main/kotlin/com/tencent/bkrepo/common/artifact/cache/config/ArtifactPreloadProperties.kt
@@ -85,4 +85,14 @@ data class ArtifactPreloadProperties(
      * 是否仅模拟预加载，为true时不执行加载计划，仅输出一条日志
      */
     var mock: Boolean = false,
+    /**
+     * 是否在缓存保留时生成预加载计划
+     *
+     * 系统中存在缓存保留策略时可能会由于保留缓存导致较大的空间占用，此时可以通过配置预加载策略，仅在需要时才将文件加载到缓存中减少空间占用，
+     * 但是由于直接移除缓存保留策略可能影响系统稳定性，不移除保留策略会导致缓存文件不会被清理无法触发预加载计划生成，无法验证预加载效果，
+     * 因此增加此开关，在缓存应该被清理，但是由于保留策略导致缓存未被清理时，也能触发预加载计划生成，验证预加载效果后再移除保留策略
+     *
+     * 需要注意此开关开启的同时需要开启[mock]，仅模拟预加载验证效果而不执行实际的加载操作，需要等所有计划执行完毕才能关闭此开关与[mock]
+     */
+    var generateOnRetained: Boolean = false,
 )
diff --git a/...in/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt b/...in/kotlin/com/tencent/bkrepo/common/artifact/cache/service/impl/CacheFileEventListener.kt
@@ -30,6 +30,8 @@ package com.tencent.bkrepo.common.artifact.cache.service.impl
 import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties
 import com.tencent.bkrepo.common.artifact.cache.service.ArtifactPreloadPlanService
 import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileDeletedEvent
+import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileEventData
+import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileRetainedEvent
 import org.slf4j.LoggerFactory
 import org.springframework.context.event.EventListener
 import org.springframework.scheduling.annotation.Async
@@ -49,10 +51,26 @@ open class CacheFileEventListener(
     @EventListener(CacheFileDeletedEvent::class)
     open fun onCacheFileDeleted(event: CacheFileDeletedEvent) {
         if (properties.enabled && event.data.size >= properties.minSize.toBytes()) {
-            with(event.data) {
-                logger.info("try generate preload plan for sha256[${sha256}], fullPath[$fullPath], size[$size")
-                preloadPlanService.generatePlan(credentials.key, sha256)
-            }
+            generatePreloadPlan(event.data)
+        }
+    }
+
+    /**
+     * 缓存被保留时判断是否需要创建预加载执行计划
+     */
+    @Async
+    @EventListener(CacheFileRetainedEvent::class)
+    open fun onCacheFileDeleted(event: CacheFileRetainedEvent) {
+        if (properties.enabled && event.data.size >= properties.minSize.toBytes() && properties.generateOnRetained) {
+            generatePreloadPlan(event.data)
+        }
+    }
+
+
+    private fun generatePreloadPlan(data: CacheFileEventData) {
+        with(data) {
+            logger.info("try generate preload plan for sha256[${sha256}], fullPath[$fullPath], size[$size")
+            preloadPlanService.generatePlan(credentials.key, sha256)
         }
     }
 

diff --git a/...rvice/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt b/...rvice/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt
@@ -229,19 +229,30 @@ class CacheStorageService(
         filename: String,
         credentials: StorageCredentials,
     ): Boolean {
+        val cacheFilePath = "${credentials.cache.path}$path$filename"
         return if (doExist(path, filename, credentials)) {
-            val cacheFilePath = "${credentials.cache.path}$path$filename"
             val size = File(cacheFilePath).length()
             getCacheClient(credentials).delete(path, filename)
             cacheFileEventPublisher.publishCacheFileDeletedEvent(path, filename, size, credentials)
-            logger.info("Cache [${credentials.cache.path}/$path/$filename] was deleted")
+            logger.info("Cache [$cacheFilePath] was deleted")
             true
         } else {
-            logger.info("Cache file[${credentials.cache.path}/$path/$filename] was not in storage")
+            logger.info("Cache file[$cacheFilePath] was not in storage")
             false
         }
     }
 
+    /**
+     * 判断缓存文件是否存在
+     */
+    fun cacheExists(
+        path: String,
+        filename: String,
+        credentials: StorageCredentials,
+    ): Boolean {
+        return getCacheClient(credentials).exist(path, filename)
+    }
+
     /**
      * 获取存储的缓存目录健康状态
      */

diff --git a/.../main/kotlin/com/tencent/bkrepo/common/storage/core/cache/event/CacheFileRetainedEvent.kt b/.../main/kotlin/com/tencent/bkrepo/common/storage/core/cache/event/CacheFileRetainedEvent.kt
@@ -0,0 +1,30 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.storage.core.cache.event
+
+data class CacheFileRetainedEvent(val data: CacheFileEventData)
diff --git a/...rc/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/cleanup/CleanupFileVisitor.kt b/...rc/main/kotlin/com/tencent/bkrepo/common/storage/filesystem/cleanup/CleanupFileVisitor.kt
@@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.artifact.constant.SHA256_STR_LENGTH
 import com.tencent.bkrepo.common.storage.core.FileStorage
 import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileDeletedEvent
 import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileEventData
+import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileRetainedEvent
 import com.tencent.bkrepo.common.storage.core.locator.FileLocator
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 import com.tencent.bkrepo.common.storage.filesystem.ArtifactFileVisitor
@@ -78,20 +79,29 @@ class CleanupFileVisitor(
             val file = filePath.toFile()
             val expired = fileExpireResolver.isExpired(file)
             val retain = fileRetainResolver?.retain(file) ?: false
-            if (expired && !retain && !isNFSTempFile(filePath)) {
-                if (isTempFile || existInStorage(filePath)) {
-                    rateLimiter.acquire()
-                    Files.delete(filePath)
-                    result.cleanupFile += 1
-                    result.cleanupSize += size
-                    deleted = true
-                    onFileCleaned(filePath, size)
-                    logger.info("Clean up file[$filePath], size[$size], summary: $result")
+
+            var shouldDelete = expired && !isNFSTempFile(filePath)
+            if (shouldDelete && !isTempFile) {
+                val existInStorage = existInStorage(filePath)
+                if (!existInStorage) {
+                    logger.info("cache file[${filePath}] not exists in storage[${credentials.key}]")
                 }
+                shouldDelete = existInStorage
+            }
+
+            if (shouldDelete && !retain) {
+                rateLimiter.acquire()
+                Files.delete(filePath)
+                result.cleanupFile += 1
+                result.cleanupSize += size
+                deleted = true
+                onFileCleaned(filePath, size)
+                logger.info("Clean up file[$filePath], size[$size], summary: $result")
             }
-            if (expired && retain) {
+            if (shouldDelete && retain) {
                 result.retainFile += 1
                 result.retainSize += size
+                onFileRetained(filePath, size)
             }
         } catch (ignored: Exception) {
             logger.error("Clean file[${filePath.fileName}] error.", ignored)
@@ -192,20 +202,26 @@ class CleanupFileVisitor(
     }
 
     private fun onFileCleaned(filePath: Path, size: Long) {
-        val fileName = filePath.fileName.toString()
         val event = FileDeletedEvent(
             credentials = credentials,
             rootPath = rootPath.toString(),
             fullPath = filePath.toString(),
         )
-        if (rootPath == credentials.cache.path.toPath() && filePath.fileName.toString().length == SHA256_STR_LENGTH) {
-            val data = CacheFileEventData(credentials, fileName, filePath.toString(), size)
+        if (isCacheFile(filePath)) {
+            val data = buildCacheFileEventData(filePath, size)
             publisher.publishEvent(CacheFileDeletedEvent(data))
         }
 
         publisher.publishEvent(event)
     }
 
+    private fun onFileRetained(filePath: Path, size: Long) {
+        if (isCacheFile(filePath)) {
+            val data = buildCacheFileEventData(filePath, size)
+            publisher.publishEvent(CacheFileRetainedEvent(data))
+        }
+    }
+
     private fun onFileSurvived(filePath: Path) {
         val event = FileSurvivedEvent(
             credentials = credentials,
@@ -215,6 +231,15 @@ class CleanupFileVisitor(
         publisher.publishEvent(event)
     }
 
+    private fun buildCacheFileEventData(filePath: Path, size: Long): CacheFileEventData {
+        val fileName = filePath.fileName.toString()
+        return CacheFileEventData(credentials, fileName, filePath.toString(), size)
+    }
+
+    private fun isCacheFile(filePath: Path): Boolean {
+        return rootPath == credentials.cache.path.toPath() && filePath.fileName.toString().length == SHA256_STR_LENGTH
+    }
+
     companion object {
         private val logger = LoggerFactory.getLogger(JOB_LOGGER_NAME)
         private const val permitsPerSecond = 30.0

diff --git a/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/StorageCacheIndexerCustomizer.kt b/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/cache/StorageCacheIndexerCustomizer.kt
@@ -30,13 +30,16 @@ package com.tencent.bkrepo.job.batch.task.cache
 import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY
 import com.tencent.bkrepo.common.storage.core.StorageService
 import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService
+import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileEventData
+import com.tencent.bkrepo.common.storage.core.cache.event.CacheFileRetainedEvent
 import com.tencent.bkrepo.common.storage.core.cache.indexer.IndexerCustomizer
 import com.tencent.bkrepo.common.storage.core.cache.indexer.StorageCacheIndexer
 import com.tencent.bkrepo.common.storage.core.cache.indexer.listener.StorageEldestRemovedListener
 import com.tencent.bkrepo.common.storage.core.cache.indexer.metrics.StorageCacheIndexerMetrics
 import com.tencent.bkrepo.common.storage.core.locator.FileLocator
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 import com.tencent.bkrepo.common.storage.filesystem.cleanup.FileRetainResolver
+import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
 
 @Component
@@ -45,11 +48,14 @@ class StorageCacheIndexerCustomizer(
     private val fileLocator: FileLocator,
     private val storageService: StorageService,
     private val storageCacheIndexerMetrics: StorageCacheIndexerMetrics? = null,
+    private val publisher: ApplicationEventPublisher,
 ) : IndexerCustomizer<String, Long> {
     override fun customize(indexer: StorageCacheIndexer<String, Long>, credentials: StorageCredentials) {
         if (storageService is CacheStorageService) {
             indexer.addEldestRemovedListener(
-                EldestRemovedListener(credentials, fileLocator, storageService, storageCacheIndexerMetrics, resolver)
+                EldestRemovedListener(
+                    credentials, fileLocator, storageService, storageCacheIndexerMetrics, resolver, publisher
+                )
             )
         }
     }
@@ -63,11 +69,21 @@ class StorageCacheIndexerCustomizer(
         storageService: CacheStorageService,
         storageCacheIndexerMetrics: StorageCacheIndexerMetrics?,
         private val resolver: FileRetainResolver,
+        private val publisher: ApplicationEventPublisher,
     ) : StorageEldestRemovedListener(storageCredentials, fileLocator, storageService, storageCacheIndexerMetrics) {
         override fun onEldestRemoved(key: String, value: Long) {
             if (!resolver.retain(key)) {
                 super.onEldestRemoved(key, value)
             } else {
+                // publish retained event
+                val path = fileLocator.locate(key)
+                if (storageService.cacheExists(path, key, storageCredentials)) {
+                    val fullPath = "${storageCredentials.cache.path}$path$key"
+                    val data = CacheFileEventData(storageCredentials, key, fullPath, value)
+                    publisher.publishEvent(CacheFileRetainedEvent(data))
+                }
+
+                // metrics
                 val storageKey = storageCredentials.key ?: DEFAULT_STORAGE_KEY
                 storageCacheIndexerMetrics?.evicted(storageKey, value, false)
             }