Log retention of curve cluster service

internal/tools/config.json

@@ -0,0 +1,6 @@
+{
+  "interval": 0,
+  "logDirectory": "/tmp/curvefs/logs",
+  "compressOrClean": 1,
+  "maxTarFile": 7
+}

internal/tools/log_rotate.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+
+############################  GLOBAL VARIABLES
+g_color_yellow=$(printf '\033[33m')
+g_color_red=$(printf '\033[31m')
+g_color_normal=$(printf '\033[0m')
+
+############################  BASIC FUNCTIONS
+msg() {
+    printf '%b' "${1}" >&2
+}
+
+success() {
+    msg "${g_color_yellow}[✔]${g_color_normal} ${1}${2}"
+}
+
+die() {
+    msg "${g_color_red}[✘]${g_color_normal} ${1}${2}"
+    exit 1
+}
+
+############################ FUNCTIONS
+load_config() {
+    # Specify JSON file path
+    json_file="config.json"
+
+    # Check if the file exists
+    if [ ! -f "$json_file" ]; then
+        die "json file ${json_file} does not exist.\n"
+    fi
+
+    # Read variables in JSON file
+    interval=$(jq -r '.interval' "$json_file")
+    log_directory=$(jq -r '.logDirectory' "$json_file")
+    compress_or_clean=$(jq -r '.compressOrClean' "$json_file")
+    max_tar_file=$(jq -r '.maxTarFile' "$json_file")
+
+}
+
+precheck() {
+    # Check whether interval is greater than or equal to 0
+    if [[ ! ($interval =~ ^[0-9]+$) ]]; then
+        die "The interval parameter is invalid.\n"
+    elif [[ ! ($interval -ge 0) ]]; then
+        die "The interval parameter is invalid.\n"
+    fi
+
+    # Check if log_directory is an absolute path
+    absolute_path_regex="^\/([a-zA-Z0-9-]+\/)*[a-zA-Z0-9-]+\/?$"
+    if ! [[ $log_directory =~ $absolute_path_regex ]]; then
+        die "The log_directory parameter is invalid.\n"
+    fi
+
+
+     # Check if compress_or_clean is equal to 0 or equal to 1
+    if [[ ! ($compress_or_clean =~ ^[0-9]+$) ]]; then
+        die "The compress_or_clean parameter is invalid.\n"
+    elif [[ ! ($compress_or_clean -eq 0 || $compress_or_clean -eq 1) ]]; then
+        die "The compress_or_clean parameter is invalid.\n"
+    fi
+
+    # Check if max_tar_file is greater than or equal to 1
+    if [[ ! ($max_tar_file =~ ^[0-9]+$) ]]; then
+        die "The max_tar_file parameter is invalid.\n"
+    elif [[ ! ($max_tar_file -ge 1) ]]; then
+        die "The max_tar_file parameter is invalid.\n"
+    fi
+}
+
+check_and_delete_tar_files() {
+    local folder="$1"
+
+    while true; do
+        # Get the count of .tar.gz files
+        local file_count=$(find "$folder" -maxdepth 1 -type f -name "*.tar.gz" | wc -l)
+
+        # Check if file count exceeds the limit
+        if [ "$file_count" -gt "$max_tar_file" ]; then
+            # Delete the oldest file
+            oldest_file=$(ls -t "$folder"/*.tar.gz | tail -n 1)
+            rm "$oldest_file"
+            msg "Deleted oldest tar file: $oldest_file\n"
+        else
+            msg "Tar file count within limit: $file_count\n"
+            break
+        fi
+    done
+}
+
+delete_expired_tar_files() {
+    local folder="$1"
+    find "$folder" -name "*.tar.gz" -type f -mtime +$interval -exec rm {} \;
+    if [ $? -eq 0 ]; then
+        msg "Clean all expired tar files in $folder\n"
+    fi
+}
+
+precompress_log() {
+    # Determine whether the file exists and has content
+    if [ -s "$file" ]; then
+        # Get the modification time of a file
+        modification_time=$(stat -c %Y "$file")
+
+        # Get the timestamp of a specified date
+        specified_timestamp=$(date -d "$day_before" +%s)
+
+        # Determine whether the file modification time is before the specified date
+        if [ "$modification_time" -lt "$specified_timestamp" ]; then
+            cat "$file" > "$dir_path/log_before_$day_before.txt"
+            > "$file"
+            tar -czvf "$dir_path/log_before_$day_before.tar.gz" -C "$dir_path" "log_before_$day_before.txt"
+            rm "$dir_path/log_before_$day_before.txt"
+            check_and_delete_tar_files "$dir_path"
+            precompress=true
+        fi
+    else
+        nodata=true
+    fi
+}
+
+retention_log() {
+    match_string=".log"
+    current_date=$(date +%s)
+    day_before=$(date -d "$interval days ago" +%Y-%m-%d)
+    find_files=$(find "$log_directory" -type f -name "*$match_string*")
+
+    file_count=$(echo "$find_files" | wc -l)
+    if [ $file_count -lt 1 ]; then
+        die "Log file not found, please check whether the log file path is correct.\n"
+    fi
+
+    for file in $find_files; do
+        # Get the file name and determine whether it is the target log
+        filename=$(basename -- "$file")
+        dir_path=$(dirname -- "$file")
+        precompress=false
+        nodata=false
+        precompress_log
+
+        if [ "$nodata" = true ]; then
+            msg "No data in $file, skipping...\n\n"
+            continue
+        fi
+
+        regex="("
+        regex+=$day_before
+        regex+=")"
+        matched_line=$(grep -m 1 -P -n "$regex" "$file" | cut -d ':' -f 1)
+
+        if [ "$compress_or_clean" -eq 1 ]; then
+            # Perform compression operation
+            if [[ ! -z "$matched_line" ]]; then
+                # Extract the data before the matching line, use the tar command to package 
+                # and compress the data, and delete the temporary files
+                head -n "$((matched_line - 1))" "$file" > "$dir_path/log_before_$day_before.txt"
+                tar -czvf "$dir_path/log_before_$day_before.tar.gz" -C "$dir_path" "log_before_$day_before.txt"
+                rm "$dir_path/log_before_$day_before.txt"
+
+                # Extract matching lines and subsequent data to filtered.txt, delete the original file, 
+                # and rename filtered.txt to the original file name
+                sed -n "$matched_line,\$p" "$file" > $dir_path/filtered.txt
+                rm "$file"
+                mv $dir_path/filtered.txt "$file"
+
+                check_and_delete_tar_files "$dir_path"
+                success "Compress log in $file\n\n"
+            elif [ "$precompress" = true ]; then
+                success "Compress log in $file\n\n"
+            else
+                msg "No data was matched in $file. The following are the first 10 lines of the log file. Please change the date and try deleting again:\n" 
+                head "$file"
+                msg "Compress log failed. Try setting the interval parameter to 0.\n\n"
+            fi
+        else
+            # Perform cleanup operations
+            delete_expired_tar_files "$dir_path"
+            if [[ ! -z "$matched_line" ]]; then
+                sed -n "$matched_line,\$p" "$file" > $dir_path/filtered.txt
+                rm "$file"
+                mv $dir_path/filtered.txt "$file"
+                msg "Clean expired log in $file\n\n"
+            elif [ "$precompress" = true ]; then
+                success "Clean expired log in $file\n\n"
+            else
+                msg "No data was matched in $file. The following are the first 10 lines of the log file. Please change the date and try deleting again:\n" 
+                head "$file"
+                msg "Compress log failed. Try setting the interval parameter to 0.\n\n"
+
+            fi
+        fi 
+
+    done    
+}
+
+main() {
+    load_config
+    precheck
+    retention_log
+}
+
+############################  MAIN()
+main
+

internal/tools/log_rotate.txt

@@ -0,0 +1,23 @@
+log_rotate.sh文件使用说明：
+
+1. 执行该文件之前，需要使用chmod +x命令给该文件添加执行权限。
+2. 该程序须自定义参数来执行自定义日志保留计划，可以在config.json定义相关参数。
+3. interval参数为时间间隔，压缩或删除在当前日期的时间间隔之前的日志。
+4. logDirectory参数为日志文件夹的绝对路径。
+5. compressOrClean参数，等于0表示执行压缩操作，等于1表示删除操作。
+6. maxTarFile参数指定最多保留的压缩日志文件。
+
+
+提示：此命令须在工作节点上使用，用于压缩或删除curve工作日志。
+----------------------------------------------
+Instructions for using log_rotate.sh file:
+
+1. Before executing the file, you need to use the chmod +x command to add execution permissions to the file.
+2. The program requires customized parameters to execute a customized log retention plan. Relevant parameters can be defined in config.json.
+3. The interval parameter is the time interval, compress or delete logs before the time interval of the current date.
+4. The logDirectory parameter is the absolute path of the log folder.
+5. compressOrClean parameter, equal to 0 means performing compression operation, equal to 1 means deletion operation.
+6. The maxTarFile parameter specifies the maximum number of compressed log files to retain.
+
+
+Tip: This command must be used on the working node to compress or delete the curve working log.