diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 921562a2f..3d6c73896 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -181,6 +181,14 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' + - name: Create zip file + run: | + zip -r data.zip data/A/ data/B/ data/C/ + + - name: Split entities json + run: | + ./devscript/split.sh data/B/entities_json/ + - name: Commit and push uses: EndBug/add-and-commit@v9 with: @@ -188,10 +196,6 @@ jobs: message: 'Update ${{ env.VERSION }}' push: true - - name: Create zip file - run: | - zip -r data.zip data/A/ data/B/ data/C/ - - name: Publish release uses: "marvinpinto/action-automatic-releases@latest" with: diff --git a/devscript/merge.sh b/devscript/merge.sh new file mode 100644 index 000000000..17dfc2a76 --- /dev/null +++ b/devscript/merge.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Check if the folder path is provided as an argument +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +folder_path="$1" + +# Check if the folder exists +if [ ! -d "$folder_path" ]; then + echo "Folder '$folder_path' does not exist." + exit 1 +fi + +# Get a list of split files in the folder +split_files=("$folder_path"/*.part*) + +# Iterate through each split file +for split_file in "${split_files[@]}"; do + # Check if it's a split file + if [ -f "$split_file" ]; then + # Determine the original file name without the ".part" suffix + original_file="${split_file%.part*}" + echo "Merging '$split_file' into '$original_file'..." + cat "$split_file" >> "$original_file.json" + echo "Merged!" + + # Remove the split part file after merging + echo "Removing '$split_file'" + rm "$split_file" + echo "Removed!" + + fi +done + +echo "Merging completed." diff --git a/devscript/split.sh b/devscript/split.sh new file mode 100644 index 000000000..ad4862bab --- /dev/null +++ b/devscript/split.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Check if the folder path is provided as an argument +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +folder_path="$1" + +# Check if the folder exists +if [ ! -d "$folder_path" ]; then + echo "Folder '$folder_path' does not exist." + exit 1 +fi + +# Get a list of files in the folder +file_list=("$folder_path"/*.json) + +# Iterate through each file in the folder +for file in "${file_list[@]}"; do + # Check if it's a regular file + if [ -f "$file" ]; then + # Split the file into smaller parts (adjust the options as needed + file_no_extension=$(echo "${file%.*}") + echo "Spliting '$file' into smaller parts..." + split -d --bytes=50M --additional-suffix=.json "$file" "$file_no_extension.part" + echo "Splited!" + + # Remove file after split + echo "Removing '$file'" + rm $file + echo "Removed!" + + fi +done + +echo "Splitting completed."