From f1f03a92a4e8062e26d2e5cadfb10f0170663f7c Mon Sep 17 00:00:00 2001 From: Khant Zaw Hein Date: Sun, 1 Dec 2024 23:58:56 +0700 Subject: [PATCH] Update auto-updater.yml --- .github/workflows/auto-updater.yml | 68 +++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/.github/workflows/auto-updater.yml b/.github/workflows/auto-updater.yml index 5e1155d..8a58fe8 100644 --- a/.github/workflows/auto-updater.yml +++ b/.github/workflows/auto-updater.yml @@ -13,15 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - # Checkout the repository + # Previous steps remain the same until the Filter Private IPs step - name: Checkout repository uses: actions/checkout@v3 - # Download the source file - name: Download source file run: curl -s -o trackers_all_ip.txt https://raw.githubusercontent.com/ngosang/trackerslist/refs/heads/master/trackers_all_ip.txt - # Download the source file with trackers - name: Download source file with trackers 1 run: curl -s -o trackers_all_from_src1.txt https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt @@ -30,34 +28,26 @@ jobs: - name: Concatenate custom hosts run: | - # Concatenate custom_hosts.txt to trackers_all.txt cat custom_hosts.txt trackers_all_from_src1.txt trackers_all_from_src2.txt > trackers_all.txt echo "Combined trackers_all.txt with custom_hosts.txt" cat trackers_all.txt - # Extract only the IPs from trackers_all_ip.txt - name: Extract IP addresses from the IP file run: | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' trackers_all_ip.txt >> extracted_ips_raw.txt echo "Extracted IPs from file:" cat extracted_ips_raw.txt - # Extract hostnames from the URLs and resolve to IP addresses - name: Extract IP addresses from hosts in trackers_all.txt run: | echo "Extracting IPs from the host list..." - - # Extract hostnames from :// URLs grep -oP '(?<=://)[^:/]+' trackers_all.txt > hostnames.txt - # Resolve each hostname to IP addresses and save it - while IFS= read -r host; do ips=$(dig +short $host) if [ -n "$ips" ]; then echo "Resolved $host to the following IPs:" for ip in $ips; do - # Check if the resolved IP is a valid IPv4 address if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then echo "$ip" echo "$ip" >> host_to_ips.txt @@ -70,30 +60,68 @@ jobs: fi done < hostnames.txt - # Combine new IPs with host to IP mappings - - name: Combine IPs run: | - # Combine new extracted IPs with the previous list and ensure uniqueness cat extracted_ips_raw.txt host_to_ips.txt > extracted_ips_unfiltered.txt echo "Combined unfiltered unique IPs:" cat extracted_ips_unfiltered.txt - - name: Filter Private IPs + - name: Download and prepare Cloudflare IP ranges + run: | + # Download Cloudflare IPv4 ranges + curl -s https://www.cloudflare.com/ips-v4/ > cloudflare_ips.txt + # Clean up the file to ensure one CIDR per line + sed -i 's/<[^>]*>//g' cloudflare_ips.txt + sed -i 's/^[[:space:]]*//g' cloudflare_ips.txt + sed -i 's/[[:space:]]*$//g' cloudflare_ips.txt + sed -i '/^$/d' cloudflare_ips.txt + echo "Downloaded Cloudflare IP ranges:" + cat cloudflare_ips.txt + + - name: Install ipcalc + run: sudo apt-get update && sudo apt-get install -y ipcalc + + - name: Filter Private and Cloudflare IPs run: | - # Filter out private IPs - grep -vE '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|169\.254\.|224\.|239\.|240\.|255\.)' extracted_ips_unfiltered.txt >> extracted_ips.txt - echo "Filtered out private IPs:" + # First filter out private IPs + grep -vE '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|169\.254\.|224\.|239\.|240\.|255\.)' extracted_ips_unfiltered.txt > temp_ips.txt + + # Then filter out Cloudflare IPs + while IFS= read -r cf_range; do + # Skip empty lines or invalid CIDR notation + [[ -z "$cf_range" ]] && continue + [[ ! "$cf_range" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$ ]] && continue + + # Convert CIDR to network and broadcast IPs for comparison + network=$(ipcalc "$cf_range" | grep "Network:" | awk '{print $2}') + broadcast=$(ipcalc "$cf_range" | grep "Broadcast:" | awk '{print $2}') + + # Convert IPs to numbers for comparison + network_num=$(echo "$network" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}') + broadcast_num=$(echo "$broadcast" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}') + + # Filter out IPs in the Cloudflare range + while IFS= read -r ip; do + ip_num=$(echo "$ip" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}') + if [ "$ip_num" -lt "$network_num" ] || [ "$ip_num" -gt "$broadcast_num" ]; then + echo "$ip" >> extracted_ips.txt + else + echo "Filtered out Cloudflare IP: $ip" + fi + done < temp_ips.txt + > temp_ips.txt # Clear the temporary file for the next iteration + done < cloudflare_ips.txt + + rm temp_ips.txt + echo "Filtered IPs:" cat extracted_ips.txt - # Ensure uniqueness of the extracted IPs - name: Ensure uniqueness of extracted IPs run: | sort -u extracted_ips.txt -o extracted_ips.txt echo "Unique extracted IPs:" cat extracted_ips.txt - # Commit and push the combined extracted IPs file - name: Commit and push changes env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}