Split files before commit

.github/workflows/workflow.yml

@@ -181,17 +181,21 @@ jobs:
         env:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
 
+      - name: Create zip file
+        run: |
+            zip -r data.zip data/A/ data/B/ data/C/
+
+      - name: Split entities json
+        run: |
+          ./devscript/split.sh data/B/entities_json/
+
       - name: Commit and push
         uses: EndBug/add-and-commit@v9
         with:
           add: '["data/A/", "data/B/", "data/C/"]'
           message: 'Update ${{ env.VERSION }}'
           push: true
 
-      - name: Create zip file
-        run: |
-            zip -r data.zip data/A/ data/B/ data/C/
-
       - name: Publish release
         uses: "marvinpinto/action-automatic-releases@latest"
         with:

devscript/merge.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Check if the folder path is provided as an argument
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 <folder_path>"
+  exit 1
+fi
+
+folder_path="$1"
+
+# Check if the folder exists
+if [ ! -d "$folder_path" ]; then
+  echo "Folder '$folder_path' does not exist."
+  exit 1
+fi
+
+# Get a list of split files in the folder
+split_files=("$folder_path"/*.part*)
+
+# Iterate through each split file
+for split_file in "${split_files[@]}"; do
+  # Check if it's a split file
+  if [ -f "$split_file" ]; then
+    # Determine the original file name without the ".part" suffix
+    original_file="${split_file%.part*}"
+    echo "Merging '$split_file' into '$original_file'..."
+    cat "$split_file" >> "$original_file.json"
+    echo "Merged!"
+
+    # Remove the split part file after merging
+    echo "Removing '$split_file'"
+    rm "$split_file"
+    echo "Removed!"
+
+  fi
+done
+
+echo "Merging completed."

devscript/split.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Check if the folder path is provided as an argument
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 <folder_path>"
+  exit 1
+fi
+
+folder_path="$1"
+
+# Check if the folder exists
+if [ ! -d "$folder_path" ]; then
+  echo "Folder '$folder_path' does not exist."
+  exit 1
+fi
+
+# Get a list of files in the folder
+file_list=("$folder_path"/*.json)
+
+# Iterate through each file in the folder
+for file in "${file_list[@]}"; do
+  # Check if it's a regular file
+  if [ -f "$file" ]; then
+    # Split the file into smaller parts (adjust the options as needed
+	  file_no_extension=$(echo "${file%.*}")
+    echo "Spliting '$file' into smaller parts..."
+    split -d --bytes=50M --additional-suffix=.json "$file" "$file_no_extension.part" 
+    echo "Splited!"
+
+    # Remove file after split
+    echo "Removing '$file'"
+	  rm $file
+    echo "Removed!"
+
+  fi
+done
+
+echo "Splitting completed."