Update auto-updater.yml #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Extract IPs from URL | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "*/30 * * * *" | |
| jobs: | |
| extract-ips: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Previous steps remain the same until the Filter Private IPs step | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Download source file | |
| run: curl -s -o trackers_all_ip.txt https://raw.githubusercontent.com/ngosang/trackerslist/refs/heads/master/trackers_all_ip.txt | |
| - name: Download source file with trackers 1 | |
| run: curl -s -o trackers_all_from_src1.txt https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt | |
| - name: Download source file with trackers 2 | |
| run: curl -s -o trackers_all_from_src2.txt https://cf.trackerslist.com/all.txt | |
| - name: Concatenate custom hosts | |
| run: | | |
| cat custom_hosts.txt trackers_all_from_src1.txt trackers_all_from_src2.txt > trackers_all.txt | |
| echo "Combined trackers_all.txt with custom_hosts.txt" | |
| cat trackers_all.txt | |
| - name: Extract IP addresses from the IP file | |
| run: | | |
| grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' trackers_all_ip.txt >> extracted_ips_raw.txt | |
| echo "Extracted IPs from file:" | |
| cat extracted_ips_raw.txt | |
| - name: Extract IP addresses from hosts in trackers_all.txt | |
| run: | | |
| echo "Extracting IPs from the host list..." | |
| grep -oP '(?<=://)[^:/]+' trackers_all.txt > hostnames.txt | |
| while IFS= read -r host; do | |
| ips=$(dig +short $host) | |
| if [ -n "$ips" ]; then | |
| echo "Resolved $host to the following IPs:" | |
| for ip in $ips; do | |
| if [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then | |
| echo "$ip" | |
| echo "$ip" >> host_to_ips.txt | |
| else | |
| echo "Skipping non-IP value: $ip" | |
| fi | |
| done | |
| else | |
| echo "Failed to resolve $host" | |
| fi | |
| done < hostnames.txt | |
| - name: Combine IPs | |
| run: | | |
| cat extracted_ips_raw.txt host_to_ips.txt > extracted_ips_unfiltered.txt | |
| echo "Combined unfiltered unique IPs:" | |
| cat extracted_ips_unfiltered.txt | |
| - name: Download and prepare Cloudflare IP ranges | |
| run: | | |
| # Download Cloudflare IPv4 ranges | |
| curl -s https://www.cloudflare.com/ips-v4/ > cloudflare_ips.txt | |
| # Clean up the file to ensure one CIDR per line | |
| sed -i 's/<[^>]*>//g' cloudflare_ips.txt | |
| sed -i 's/^[[:space:]]*//g' cloudflare_ips.txt | |
| sed -i 's/[[:space:]]*$//g' cloudflare_ips.txt | |
| sed -i '/^$/d' cloudflare_ips.txt | |
| echo "Downloaded Cloudflare IP ranges:" | |
| cat cloudflare_ips.txt | |
| - name: Install ipcalc | |
| run: sudo apt-get update && sudo apt-get install -y ipcalc | |
| - name: Filter Private and Cloudflare IPs | |
| run: | | |
| # First filter out private IPs | |
| grep -vE '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|169\.254\.|224\.|239\.|240\.|255\.)' extracted_ips_unfiltered.txt > temp_ips.txt | |
| # Then filter out Cloudflare IPs | |
| while IFS= read -r cf_range; do | |
| # Skip empty lines or invalid CIDR notation | |
| [[ -z "$cf_range" ]] && continue | |
| [[ ! "$cf_range" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$ ]] && continue | |
| # Convert CIDR to network and broadcast IPs for comparison | |
| network=$(ipcalc "$cf_range" | grep "Network:" | awk '{print $2}') | |
| broadcast=$(ipcalc "$cf_range" | grep "Broadcast:" | awk '{print $2}') | |
| # Convert IPs to numbers for comparison | |
| network_num=$(echo "$network" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}') | |
| broadcast_num=$(echo "$broadcast" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}') | |
| # Filter out IPs in the Cloudflare range | |
| while IFS= read -r ip; do | |
| ip_num=$(echo "$ip" | awk -F. '{print ($1*256^3)+($2*256^2)+($3*256)+$4}') | |
| if [ "$ip_num" -lt "$network_num" ] || [ "$ip_num" -gt "$broadcast_num" ]; then | |
| echo "$ip" >> extracted_ips.txt | |
| else | |
| echo "Filtered out Cloudflare IP: $ip" | |
| fi | |
| done < temp_ips.txt | |
| > temp_ips.txt # Clear the temporary file for the next iteration | |
| done < cloudflare_ips.txt | |
| rm temp_ips.txt | |
| echo "Filtered IPs:" | |
| cat extracted_ips.txt | |
| - name: Ensure uniqueness of extracted IPs | |
| run: | | |
| sort -u extracted_ips.txt -o extracted_ips.txt | |
| echo "Unique extracted IPs:" | |
| cat extracted_ips.txt | |
| - name: Commit and push changes | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git add extracted_ips.txt | |
| git commit -m "Update extracted IPs" || echo "No changes to commit" && exit 0 | |
| git push |