🎨 Automatic purge for local data repo #13091

siyuan-note · Nov 16, 2024 · 6a442ad · 6a442ad
1 parent 93b687a
commit 6a442ad
Show file tree

Hide file tree

Showing 23 changed files with 259 additions and 15 deletions.
diff --git a/app/appearance/langs/de_DE.json b/app/appearance/langs/de_DE.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Daten-Snapshot-Aufbewahrungstage",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Daten-Snapshots pro Tag",
   "fields": "Attribut",
   "dynamicEmoji": "Dynamisches Icon",
   "backlinkContainChildren": "Enthalten Rückverweise untergeordnete Blöcke",

diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Data snapshot retention days",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Data snapshots per day",
   "fields": "Fields",
   "dynamicEmoji": "Dynamic icon",
   "backlinkContainChildren": "Do backlinks contain child blocks",

diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Días de retención de instantáneas de datos",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Número de instantáneas de datos por día",
   "fields": "Atributo",
   "dynamicEmoji": "Icono dinámico",
   "backlinkContainChildren": "¿Los enlaces inversos contienen bloques secundarios?",

diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Jours de rétention des instantanés de données",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Nombre d'instantanés de données par jour",
   "fields": "Attribut",
   "dynamicEmoji": "Icône dynamique",
   "backlinkContainChildren": "Les liens retour contiennent-ils des sous-blocs",

diff --git a/app/appearance/langs/he_IL.json b/app/appearance/langs/he_IL.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "ימי שמירת תמונות נתונים",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "מספר תמונות נתונים ביום",
   "fields": "מאפיין",
   "dynamicEmoji": "אייקון דינמי",
   "backlinkContainChildren": "האם קישורים חוזרים כוללים בלוקים משניים",

diff --git a/app/appearance/langs/it_IT.json b/app/appearance/langs/it_IT.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Giorni di conservazione degli snapshot dei dati",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Numero di snapshot dei dati al giorno",
   "fields": "Campi",
   "dynamicEmoji": "Emoji dinamica",
   "backlinkContainChildren": "I backlink contengono blocchi figli",

diff --git a/app/appearance/langs/ja_JP.json b/app/appearance/langs/ja_JP.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "データスナップショットの保持日数",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "データスナップショットの毎日の保持数",
   "fields": "属性",
   "dynamicEmoji": "動的アイコン",
   "backlinkContainChildren": "バックリンクに子ブロックを含めるかどうか",

diff --git a/app/appearance/langs/pl_PL.json b/app/appearance/langs/pl_PL.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Dni przechowywania migawek danych",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Liczba migawek danych dziennie",
   "fields": "Atrybut",
   "dynamicEmoji": "Ikona dynamiczna",
   "backlinkContainChildren": "Czy linki zwrotne zawierają bloki podrzędne",

diff --git a/app/appearance/langs/ru_RU.json b/app/appearance/langs/ru_RU.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "Срок хранения снимков данных",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "Количество снимков данных в день",
   "fields": "Атрибут",
   "dynamicEmoji": "Динамическая иконка",
   "backlinkContainChildren": "Включать ли дочерние блоки в обратные ссылки",

diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "數據快照保留天數",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "數據快照每天保留個數",
   "fields": "欄位",
   "dynamicEmoji": "動態圖標",
   "backlinkContainChildren": "反向鏈接包含子塊",

diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json
@@ -1,4 +1,6 @@
 {
+  "dataRepoAutoPurgeIndexRetentionDays": "数据快照保留天数",
+  "dataRepoAutoPurgeRetentionIndexesDaily": "数据快照每天保留个数",
   "fields": "字段",
   "dynamicEmoji": "动态图标",
   "backlinkContainChildren": "反向链接包含子块",

diff --git a/app/src/config/about.ts b/app/src/config/about.ts
@@ -142,16 +142,31 @@ export const about = {
         </button>
     </div>
 </div>
-<div class="fn__flex b3-label config__item">
-    <div class="fn__flex-1">
+<div class="b3-label fn__flex"><div class="fn__block">
+    <div>
         ${window.siyuan.languages.dataRepoPurge}
-        <div class="b3-label__text">${window.siyuan.languages.dataRepoPurgeTip}</div>
     </div>
-    <div class="fn__space"></div>
-    <button id="purgeRepo" class="b3-button b3-button--outline fn__size200 fn__flex-center">
-        <svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
-    </button>
-</div>
+    <div class="fn__hr"></div>
+    <div class="fn__flex config__item">
+        <div class="fn__flex-center fn__flex-1 ft__on-surface">${window.siyuan.languages.dataRepoPurgeTip}</div>
+        <span class="fn__space"></span>
+        <button id="purgeRepo" class="b3-button b3-button--outline fn__size200 fn__flex-center">
+            <svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
+        </button>
+    </div>
+    <div class="fn__hr"></div>
+    <div class="fn__flex config__item">
+        <div class="fn__flex-center fn__flex-1 ft__on-surface">${window.siyuan.languages.dataRepoAutoPurgeIndexRetentionDays}</div>
+        <span class="fn__space"></span>
+        <input class="b3-text-field fn__flex-center fn__size200" min="1" type="number" id="indexRetentionDays" value="${window.siyuan.config.repo.indexRetentionDays}">
+    </div>
+    <div class="fn__hr"></div>
+    <div class="fn__flex config__item">
+        <div class="fn__flex-center fn__flex-1 ft__on-surface">${window.siyuan.languages.dataRepoAutoPurgeRetentionIndexesDaily}</div>
+        <span class="fn__space"></span>
+        <input class="b3-text-field fn__flex-center fn__size200" min="1" type="number" id="retentionIndexesDaily" value="${window.siyuan.config.repo.retentionIndexesDaily}">
+    </div>
+</div></div>
 <div class="fn__flex b3-label config__item">
     <div class="fn__flex-1">
         ${window.siyuan.languages.systemLog}
@@ -208,6 +223,18 @@ ${checkUpdateHTML}
         if (window.siyuan.config.system.isInsider) {
             about.element.querySelector("#isInsider").innerHTML = "<span class='ft__secondary'>Insider Preview</span>";
         }
+        const indexRetentionDaysElement = about.element.querySelector("#indexRetentionDays") as HTMLInputElement;
+        indexRetentionDaysElement.addEventListener("change", () => {
+            fetchPost("/api/repo/setRepoIndexRetentionDays", {days: parseInt(indexRetentionDaysElement.value)}, () => {
+                window.siyuan.config.repo.indexRetentionDays = parseInt(indexRetentionDaysElement.value);
+            });
+        });
+        const retentionIndexesDailyElement = about.element.querySelector("#retentionIndexesDaily") as HTMLInputElement;
+        retentionIndexesDailyElement.addEventListener("change", () => {
+            fetchPost("/api/repo/setRetentionIndexesDaily", {indexes: parseInt(retentionIndexesDailyElement.value)}, () => {
+                window.siyuan.config.repo.retentionIndexesDaily = parseInt(retentionIndexesDailyElement.value);
+            });
+        });
         const tokenElement = about.element.querySelector("#token") as HTMLInputElement;
         tokenElement.addEventListener("click", () => {
             tokenElement.select();

diff --git a/app/src/mobile/settings/about.ts b/app/src/mobile/settings/about.ts
@@ -83,6 +83,12 @@ export const initAbout = () => {
         <svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
     </button>
     <div class="b3-label__text">${window.siyuan.languages.dataRepoPurgeTip}</div>
+    <div class="fn__hr"></div>
+    <input class="b3-text-field fn__block" style="padding-right: 64px;" id="indexRetentionDays" min="1" type="number" class="b3-text-field" value="${window.siyuan.config.repo.indexRetentionDays}">
+    <div class="b3-label__text">${window.siyuan.languages.dataRepoAutoPurgeIndexRetentionDays}</div>
+    <div class="fn__hr"></div>
+    <input class="b3-text-field fn__block" style="padding-right: 64px;" id="retentionIndexesDaily" min="1" type="number" class="b3-text-field" value="${window.siyuan.config.repo.retentionIndexesDaily}">
+    <div class="b3-label__text">${window.siyuan.languages.dataRepoAutoPurgeRetentionIndexesDaily}</div>
 </div>
 <div class="b3-label">
     ${window.siyuan.languages.systemLog}
@@ -410,6 +416,18 @@ export const initAbout = () => {
                     modelMainElement.querySelector("#tokenTip").innerHTML = window.siyuan.languages.about14.replace("${token}", window.siyuan.config.api.token);
                 });
             });
+            const indexRetentionDaysElement = modelMainElement.querySelector("#indexRetentionDays") as HTMLInputElement;
+            indexRetentionDaysElement.addEventListener("change", () => {
+                fetchPost("/api/repo/setRepoIndexRetentionDays", {days: parseInt(indexRetentionDaysElement.value)}, () => {
+                    window.siyuan.config.repo.indexRetentionDays = parseInt(indexRetentionDaysElement.value);
+                });
+            });
+            const retentionIndexesDailyElement = modelMainElement.querySelector("#retentionIndexesDaily") as HTMLInputElement;
+            retentionIndexesDailyElement.addEventListener("change", () => {
+                fetchPost("/api/repo/setRetentionIndexesDaily", {indexes: parseInt(retentionIndexesDailyElement.value)}, () => {
+                    window.siyuan.config.repo.retentionIndexesDaily = parseInt(retentionIndexesDailyElement.value);
+                });
+            });
         }
     });
 };

diff --git a/app/src/types/config.d.ts b/app/src/types/config.d.ts
@@ -1110,10 +1110,18 @@ declare namespace Config {
          */
         key: string;
         /**
-         * Synchronous index timing, if it exceeds this time, the user is prompted that the index
+         * Sync index timing, if it exceeds this time, the user is prompted that the index
          * performance is degraded (unit: milliseconds)
          */
         syncIndexTiming: number;
+        /**
+         * Automatic purge for local data repo index retention days
+         */
+        indexRetentionDays: number;
+        /**
+         * Automatic purge for local data repo indexes retention daily
+         */
+        retentionIndexesDaily: number;
     }
 
     /**

diff --git a/kernel/api/repo.go b/kernel/api/repo.go
@@ -29,6 +29,40 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
+func setRepoIndexRetentionDays(c *gin.Context) {
+	ret := gulu.Ret.NewResult()
+	defer c.JSON(http.StatusOK, ret)
+
+	arg, ok := util.JsonArg(c, ret)
+	if !ok {
+		return
+	}
+	days := int(arg["days"].(float64))
+	if 1 > days {
+		days = 180
+	}
+
+	model.Conf.Repo.IndexRetentionDays = days
+	model.Conf.Save()
+}
+
+func setRetentionIndexesDaily(c *gin.Context) {
+	ret := gulu.Ret.NewResult()
+	defer c.JSON(http.StatusOK, ret)
+
+	arg, ok := util.JsonArg(c, ret)
+	if !ok {
+		return
+	}
+	indexes := int(arg["indexes"].(float64))
+	if 1 > indexes {
+		indexes = 180
+	}
+
+	model.Conf.Repo.RetentionIndexesDaily = indexes
+	model.Conf.Save()
+}
+
 func getRepoFile(c *gin.Context) {
 	// Add internal kernel API `/api/repo/getRepoFile` https://github.com/siyuan-note/siyuan/issues/10101
 

diff --git a/kernel/api/router.go b/kernel/api/router.go
@@ -390,6 +390,8 @@ func ServeAPI(ginServer *gin.Engine) {
 	ginServer.Handle("POST", "/api/repo/diffRepoSnapshots", model.CheckAuth, model.CheckAdminRole, diffRepoSnapshots)
 	ginServer.Handle("POST", "/api/repo/openRepoSnapshotDoc", model.CheckAuth, model.CheckAdminRole, openRepoSnapshotDoc)
 	ginServer.Handle("POST", "/api/repo/getRepoFile", model.CheckAuth, model.CheckAdminRole, getRepoFile)
+	ginServer.Handle("POST", "/api/repo/setRepoIndexRetentionDays", model.CheckAuth, model.CheckAdminRole, setRepoIndexRetentionDays)
+	ginServer.Handle("POST", "/api/repo/setRetentionIndexesDaily", model.CheckAuth, model.CheckAdminRole, setRetentionIndexesDaily)
 
 	ginServer.Handle("POST", "/api/riff/createRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, createRiffDeck)
 	ginServer.Handle("POST", "/api/riff/renameRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, renameRiffDeck)

diff --git a/kernel/conf/repo.go b/kernel/conf/repo.go
@@ -29,11 +29,17 @@ type Repo struct {
 	// If the data repo indexing time is greater than 12s, prompt user to purge the data repo https://github.com/siyuan-note/siyuan/issues/9613
 	// Supports configuring data sync index time-consuming prompts https://github.com/siyuan-note/siyuan/issues/9698
 	SyncIndexTiming int64 `json:"syncIndexTiming"`
+
+	// 自动清理数据仓库 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091
+	IndexRetentionDays    int `json:"indexRetentionDays"`    // 索引保留天数
+	RetentionIndexesDaily int `json:"retentionIndexesDaily"` // 每日保留索引数
 }
 
 func NewRepo() *Repo {
 	return &Repo{
-		SyncIndexTiming: 12 * 1000,
+		SyncIndexTiming:       12 * 1000,
+		IndexRetentionDays:    180,
+		RetentionIndexesDaily: 2,
 	}
 }
 

diff --git a/kernel/go.mod b/kernel/go.mod
@@ -56,7 +56,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/sashabaranov/go-openai v1.29.1
 	github.com/shirou/gopsutil/v3 v3.24.5
-	github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97
+	github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836
 	github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
 	github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
 	github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04

diff --git a/kernel/go.sum b/kernel/go.sum
@@ -340,8 +340,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
 github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
-github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97 h1:JkJG+dYDHSPCrzKXfi/yczlobfgiDSDHoS6ozRMNRNI=
-github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
+github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836 h1:+hHN29Ifk2KqdXFgFIj++3bgqEOv8SjHClmFcSvj0eE=
+github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=

diff --git a/kernel/job/cron.go b/kernel/job/cron.go
@@ -42,6 +42,7 @@ func StartCron() {
 	go every(30*time.Second, model.OCRAssetsJob)
 	go every(30*time.Second, model.FlushAssetsTextsJob)
 	go every(30*time.Second, model.HookDesktopUIProcJob)
+	go every(24*time.Hour, model.AutoPurgeRepoJob)
 }
 
 func every(interval time.Duration, f func()) {

diff --git a/kernel/model/conf.go b/kernel/model/conf.go
@@ -385,6 +385,12 @@ func InitConf() {
 	if 12000 > Conf.Repo.SyncIndexTiming {
 		Conf.Repo.SyncIndexTiming = 12 * 1000
 	}
+	if 1 > Conf.Repo.IndexRetentionDays {
+		Conf.Repo.IndexRetentionDays = 180
+	}
+	if 1 > Conf.Repo.RetentionIndexesDaily {
+		Conf.Repo.RetentionIndexesDaily = 2
+	}
 
 	if nil == Conf.Search {
 		Conf.Search = conf.NewSearch()